ready to push

22 files changed, 89 insertions, 303 deletions

diff --git a/python/pylibvw.cc b/python/pylibvw.cc
index 90bbb5c5..923b65e9 100644
--- a/python/pylibvw.cc
+++ b/python/pylibvw.cc
@@ -359,12 +359,6 @@ BOOST_PYTHON_MODULE(pylibvw) {
       .def("get_topic_prediction", &VW::get_topic_prediction, "For LDA models, returns the topic prediction for the topic id given")
       .def("get_feature_number", &VW::get_feature_number, "Returns the total number of features for this example")
 
-      // the following four are redundant with get_simplelabel_...
-      //.def("get_label", &VW::get_label, "Returns the label (simple_label only!) for this example")
-      //.def("get_importance", &VW::get_importance, "Returns the importance (simple_label only!) for this example")
-      //.def("get_initial", &VW::get_initial, "TODO")
-      //.def("get_prediction", &VW::get_prediction, "TODO")
-      
       .def("get_example_counter", &get_example_counter, "Returns the counter of total number of examples seen up to and including this one")
       .def("get_ft_offset", &get_ft_offset, "Returns the feature offset for this example (used, eg, by multiclass classification to bulk offset all features)")
       .def("get_partial_prediction", &get_partial_prediction, "Returns the partial prediction associated with this example")
@@ -372,11 +366,6 @@ BOOST_PYTHON_MODULE(pylibvw) {
       .def("get_loss", &get_loss, "Returns the loss associated with this example")
       .def("get_example_t", &get_example_t, "The total sum of importance weights up to and including this example")
       .def("get_total_sum_feat_sq", &get_total_sum_feat_sq, "The total sum of feature-value squared for this example")
-      //      .def_readwrite("revert_weight", &example::revert_weight)
-      //      .def_readwrite("test_only", &example::test_only)
-      //      .def_readwrite("end_pass", &example::end_pass)
-      //      .def_readwrite("sorted", &example::sorted)
-      //      .def_readwrite("in_use", &example::in_use)
 
       .def("num_namespaces", &ex_num_namespaces, "The total number of namespaces associated with this example")
       .def("namespace", &ex_namespace, "Get the namespace id for namespace i (for i = 0.. num_namespaces); specifically returns the ord() of the corresponding character id")
@@ -415,23 +404,23 @@ BOOST_PYTHON_MODULE(pylibvw) {
       ;
 
   py::class_<Search::predictor, predictor_ptr>("predictor", py::no_init)
-      .def("set_input", &my_set_input, "TODO")
-      .def("set_input_ldf", &my_set_input_LDF, "TODO")
-      .def("add_oracle", &my_add_oracle, "TODO")
-      .def("add_oracles", &my_add_oracles, "TODO")
-      .def("add_allowed", &my_add_allowed, "TODO")
-      .def("add_alloweds", &my_add_alloweds, "TODO")
-      .def("add_condition", &my_add_condition, "TODO")
-      .def("add_condition_range", &my_add_condition_range, "TODO")
-      .def("set_oracle", &my_set_oracle, "TODO")
-      .def("set_oracles", &my_set_oracles, "TODO")
-      .def("set_allowed", &my_set_allowed, "TODO")
-      .def("set_alloweds", &my_set_alloweds, "TODO")
-      .def("set_condition", &my_set_condition, "TODO")
-      .def("set_condition_range", &my_set_condition_range, "TODO")
-      .def("set_learner_id", &my_set_learner_id, "TODO")
-      .def("set_tag", &my_set_tag, "TODO")
-      .def("predict", &Search::predictor::predict, "TODO")
+      .def("set_input", &my_set_input, "set the input (an example) for this predictor (non-LDF mode only)")
+      .def("set_input_ldf", &my_set_input_LDF, "set the inputs (a list of examples) for this predictor (LDF mode only)")
+      .def("add_oracle", &my_add_oracle, "add an action to the current list of oracle actions")
+      .def("add_oracles", &my_add_oracles, "add a list of actions to the current list of oracle actions")
+      .def("add_allowed", &my_add_allowed, "add an action to the current list of allowed actions")
+      .def("add_alloweds", &my_add_alloweds, "add a list of actions to the current list of allowed actions")
+      .def("add_condition", &my_add_condition, "add a (tag,char) pair to the list of variables on which to condition")
+      .def("add_condition_range", &my_add_condition_range, "given (tag,len,char), add (tag,char), (tag-1,char+1), ..., (tag-len,char+len) to the list of conditionings")
+      .def("set_oracle", &my_set_oracle, "set an action as the current list of oracle actions")
+      .def("set_oracles", &my_set_oracles, "set a list of actions as the current list of oracle actions")
+      .def("set_allowed", &my_set_allowed, "set an action as the current list of allowed actions")
+      .def("set_alloweds", &my_set_alloweds, "set a list of actions as the current list of allowed actions")
+      .def("set_condition", &my_set_condition, "set a (tag,char) pair as the list of variables on which to condition")
+      .def("set_condition_range", &my_set_condition_range, "given (tag,len,char), set (tag,char), (tag-1,char+1), ..., (tag-len,char+len) as the list of conditionings")
+      .def("set_learner_id", &my_set_learner_id, "select the learner with which to make this prediction")
+      .def("set_tag", &my_set_tag, "change the tag of this prediction")
+      .def("predict", &Search::predictor::predict, "make a prediction")
       ;
   
   py::class_<Search::search, search_ptr>("search")
diff --git a/test/RunTests b/test/RunTests
index 048c4815..6df464ad 100755
--- a/test/RunTests
+++ b/test/RunTests
@@ -954,8 +954,8 @@ __DATA__
     train-sets/ref/sequence_data.ldf.test.stderr
     train-sets/ref/sequence_data.ldf.test.predict
 
-# Test 48: search sequence SPAN labeling BIO, non-ldf train
-{VW} -k -c -d train-sets/sequencespan_data --passes 20 --invariant --search_rollout oracle --search_alpha 1e-8 --search_task sequencespan --search 7 --holdout_off -f models/sequencespan_data.model
+# Test 48: search sequence SPAN labeling BIO, non-ldf train, no rollouts
+{VW} -k -c -d train-sets/sequencespan_data --passes 20 --invariant --search_rollout none --search_task sequencespan --search 7 --holdout_off -f models/sequencespan_data.model
     train-sets/ref/sequencespan_data.nonldf.train.stderr
 
 # Test 49: search sequence SPAN labeling BIO, non-ldf test
diff --git a/test/train-sets/ref/sequence_data.ldf.test-beam1.predict b/test/train-sets/ref/sequence_data.ldf.test-beam1.predict
deleted file mode 100644
index e40f56db..00000000
--- a/test/train-sets/ref/sequence_data.ldf.test-beam1.predict
+++ /dev/null
@@ -1 +0,0 @@
-5 4 3 2 1 
diff --git a/test/train-sets/ref/sequence_data.ldf.test-beam1.stderr b/test/train-sets/ref/sequence_data.ldf.test-beam1.stderr
deleted file mode 100644
index 82ebc737..00000000
--- a/test/train-sets/ref/sequence_data.ldf.test-beam1.stderr
+++ /dev/null
@@ -1,24 +0,0 @@
-only testing
-Num weight bits = 18
-learning rate = 10
-initial_t = 1
-power_t = 0.5
-predictions = sequence_data.predict
-warning: setting searn_beam=1 is kind of a weird thing to do -- just don't use a beam at all
-using no cache
-Reading datafile = train-sets/sequence_data
-num sources = 1
-average    since         example     example  current  current  current
-loss       last          counter      weight    label  predict features
-average    since      sequence         example            current label      current predicted  current   cur   cur         predic.        examples
-loss       last        counter          weight          sequence prefix        sequence prefix features  pass   pol            made          gener.
-0.000000   0.000000          1        1.000000   [5 4 3 2 1           ] [5 4 3 2 1           ]        0     0     0             105               0
-
-finished run
-number of examples per pass = 1
-passes used = 1
-weighted example sum = 1
-weighted label sum = 0
-average loss = 0
-best constant = -inf
-total feature number = 0
diff --git a/test/train-sets/ref/sequence_data.ldf.test-beam20.predict b/test/train-sets/ref/sequence_data.ldf.test-beam20.predict
deleted file mode 100644
index cad07d8e..00000000
--- a/test/train-sets/ref/sequence_data.ldf.test-beam20.predict
+++ /dev/null
@@ -1,21 +0,0 @@
-1.78814e-07	5 4 3 2 1 
-1	5 4 3 2 4 
-1	5 4 3 2 5 
-1.00594	5 4 3 2 3 
-1.0319	5 4 3 5 4 
-1.0332	5 4 3 4 3 
-1.04187	5 4 3 2 2 
-1.06594	5 4 3 3 2 
-1.1154	5 4 5 4 3 
-1.13135	5 4 4 3 2 
-1.24821	5 5 4 3 2 
-1.71179	5 4 2 1 2 
-1.71179	5 3 2 1 2 
-1.71179	5 4 3 1 2 
-1.71438	5 4 2 1 3 
-1.71438	5 3 2 1 3 
-1.71438	5 4 3 1 3 
-1.72888	5 2 1 2 1 
-1.73146	5 2 1 3 2 
-1.73258	5 4 2 1 5 
-
diff --git a/test/train-sets/ref/sequence_data.ldf.test-beam20.stderr b/test/train-sets/ref/sequence_data.ldf.test-beam20.stderr
deleted file mode 100644
index 874b4362..00000000
--- a/test/train-sets/ref/sequence_data.ldf.test-beam20.stderr
+++ /dev/null
@@ -1,23 +0,0 @@
-only testing
-Num weight bits = 18
-learning rate = 10
-initial_t = 1
-power_t = 0.5
-predictions = sequence_data.predict
-using no cache
-Reading datafile = train-sets/sequence_data
-num sources = 1
-average    since         example     example  current  current  current
-loss       last          counter      weight    label  predict features
-average    since      sequence         example            current label      current predicted  current   cur   cur         predic.        examples
-loss       last        counter          weight          sequence prefix        sequence prefix features  pass   pol            made          gener.
-3.000000   3.000000          1        1.000000   [5 4 3 2 1           ] [5 4 3 2 1           ]        0     0     0            1155               0
-
-finished run
-number of examples per pass = 1
-passes used = 1
-weighted example sum = 1
-weighted label sum = 0
-average loss = 3
-best constant = -inf
-total feature number = 0
diff --git a/test/train-sets/ref/sequence_data.nonldf.test-beam1.predict b/test/train-sets/ref/sequence_data.nonldf.test-beam1.predict
deleted file mode 100644
index e40f56db..00000000
--- a/test/train-sets/ref/sequence_data.nonldf.test-beam1.predict
+++ /dev/null
@@ -1 +0,0 @@
-5 4 3 2 1 
diff --git a/test/train-sets/ref/sequence_data.nonldf.test-beam1.stderr b/test/train-sets/ref/sequence_data.nonldf.test-beam1.stderr
deleted file mode 100644
index c28959b8..00000000
--- a/test/train-sets/ref/sequence_data.nonldf.test-beam1.stderr
+++ /dev/null
@@ -1,24 +0,0 @@
-only testing
-Num weight bits = 18
-learning rate = 10
-initial_t = 1
-power_t = 0.5
-predictions = sequence_data.predict
-warning: setting searn_beam=1 is kind of a weird thing to do -- just don't use a beam at all
-using no cache
-Reading datafile = train-sets/sequence_data
-num sources = 1
-average    since         example     example  current  current  current
-loss       last          counter      weight    label  predict features
-average    since      sequence         example            current label      current predicted  current   cur   cur         predic.        examples
-loss       last        counter          weight          sequence prefix        sequence prefix features  pass   pol            made          gener.
-0.000000   0.000000          1        1.000000   [5 4 3 2 1           ] [5 4 3 2 1           ]        0     0     0              21               0
-
-finished run
-number of examples per pass = 1
-passes used = 1
-weighted example sum = 1
-weighted label sum = 0
-average loss = 0
-best constant = -inf
-total feature number = 0
diff --git a/test/train-sets/ref/sequence_data.nonldf.test-beam20.predict b/test/train-sets/ref/sequence_data.nonldf.test-beam20.predict
deleted file mode 100644
index dbe810cb..00000000
--- a/test/train-sets/ref/sequence_data.nonldf.test-beam20.predict
+++ /dev/null
@@ -1,21 +0,0 @@
-8.34465e-07	5 4 3 2 1 
-1	5 4 3 2 4 
-1	5 4 3 2 5 
-1.02138	5 4 3 2 3 
-1.02816	5 4 3 5 4 
-1.03424	5 4 3 4 3 
-1.09347	5 4 3 2 2 
-1.10785	5 4 5 4 3 
-1.10795	5 4 3 3 2 
-1.13741	5 4 4 3 2 
-1.24793	5 5 4 3 2 
-1.76761	5 3 2 1 2 
-1.76761	5 4 3 1 2 
-1.76761	5 4 2 1 2 
-1.76948	5 3 2 1 3 
-1.76948	5 4 3 1 3 
-1.76948	5 4 2 1 3 
-1.77888	5 3 2 1 4 
-1.77889	5 4 3 1 4 
-1.77889	5 4 2 1 4 
-
diff --git a/test/train-sets/ref/sequence_data.nonldf.test-beam20.stderr b/test/train-sets/ref/sequence_data.nonldf.test-beam20.stderr
deleted file mode 100644
index 3e73329a..00000000
--- a/test/train-sets/ref/sequence_data.nonldf.test-beam20.stderr
+++ /dev/null
@@ -1,23 +0,0 @@
-only testing
-Num weight bits = 18
-learning rate = 10
-initial_t = 1
-power_t = 0.5
-predictions = sequence_data.predict
-using no cache
-Reading datafile = train-sets/sequence_data
-num sources = 1
-average    since         example     example  current  current  current
-loss       last          counter      weight    label  predict features
-average    since      sequence         example            current label      current predicted  current   cur   cur         predic.        examples
-loss       last        counter          weight          sequence prefix        sequence prefix features  pass   pol            made          gener.
-3.000000   3.000000          1        1.000000   [5 4 3 2 1           ] [5 4 3 2 1           ]        0     0     0             231               0
-
-finished run
-number of examples per pass = 1
-passes used = 1
-weighted example sum = 1
-weighted label sum = 0
-average loss = 3
-best constant = -inf
-total feature number = 0
diff --git a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam1.predict b/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam1.predict
deleted file mode 100644
index 980567ab..00000000
--- a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam1.predict
+++ /dev/null
@@ -1 +0,0 @@
-2 1 1 2 2 1 6 7 7 7 7 1 6 4 1 
diff --git a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam1.stderr b/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam1.stderr
deleted file mode 100644
index 04e25211..00000000
--- a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam1.stderr
+++ /dev/null
@@ -1,25 +0,0 @@
-only testing
-Num weight bits = 18
-learning rate = 10
-initial_t = 1
-power_t = 0.5
-predictions = sequencespan_data.predict
-warning: setting searn_beam=1 is kind of a weird thing to do -- just don't use a beam at all
-switching to BILOU encoding for sequence span labeling
-using no cache
-Reading datafile = train-sets/sequencespan_data
-num sources = 1
-average    since         example     example  current  current  current
-loss       last          counter      weight    label  predict features
-average    since      sequence         example            current label      current predicted  current   cur   cur         predic.        examples
-loss       last        counter          weight          sequence prefix        sequence prefix features  pass   pol            made          gener.
-0.000000   0.000000          1        1.000000   [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]        0     0     0              79               0
-
-finished run
-number of examples per pass = 1
-passes used = 1
-weighted example sum = 1
-weighted label sum = 0
-average loss = 0
-best constant = -inf
-total feature number = 0
diff --git a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam20.predict b/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam20.predict
deleted file mode 100644
index a7da94aa..00000000
--- a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam20.predict
+++ /dev/null
@@ -1,21 +0,0 @@
--0.437092	2 1 1 2 2 1 6 7 7 7 7 1 6 4 1 
-0.207599	2 2 1 2 2 1 6 7 7 7 7 1 6 4 1 
-0.255674	2 4 1 2 2 1 6 7 7 7 7 1 6 4 1 
-0.414854	2 1 1 2 4 1 6 7 7 7 7 1 6 4 1 
-0.469988	2 1 2 2 2 1 6 7 7 7 7 1 6 4 1 
-0.632442	2 1 1 2 2 1 6 7 7 7 7 1 6 4 2 
-0.632442	2 1 1 2 2 1 6 7 7 7 7 1 6 4 4 
-0.638117	2 1 1 2 2 1 6 7 7 7 7 1 6 4 2 
-0.641751	2 1 1 2 2 1 6 7 7 7 7 1 6 6 7 
-0.665272	2 1 1 2 2 1 6 7 7 7 7 1 6 4 4 
-0.665499	2 1 1 2 2 1 6 7 7 7 7 1 6 2 3 
-0.665499	2 1 1 2 2 1 6 7 7 7 7 1 6 2 3 
-0.665499	2 1 1 2 2 1 6 7 7 7 7 1 6 4 5 
-0.665499	2 1 1 2 2 1 6 7 7 7 7 1 6 4 5 
-0.668673	2 1 1 2 2 1 6 7 7 7 7 1 6 4 6 
-0.706717	2 1 6 2 2 1 6 7 7 7 7 1 6 4 1 
-0.733527	2 1 1 2 2 1 6 7 7 7 7 2 3 3 3 
-0.733527	2 1 1 2 2 1 6 7 7 7 7 2 3 3 3 
-0.733527	2 1 1 2 2 1 6 7 7 7 7 4 5 5 5 
-0.733527	2 1 1 2 2 1 6 7 7 7 7 4 5 5 5 
-
diff --git a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam20.stderr b/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam20.stderr
deleted file mode 100644
index 12df2b1e..00000000
--- a/test/train-sets/ref/sequencespan_data.nonldf-bilou.test-beam20.stderr
+++ /dev/null
@@ -1,24 +0,0 @@
-only testing
-Num weight bits = 18
-learning rate = 10
-initial_t = 1
-power_t = 0.5
-predictions = sequencespan_data.predict
-switching to BILOU encoding for sequence span labeling
-using no cache
-Reading datafile = train-sets/sequencespan_data
-num sources = 1
-average    since         example     example  current  current  current
-loss       last          counter      weight    label  predict features
-average    since      sequence         example            current label      current predicted  current   cur   cur         predic.        examples
-loss       last        counter          weight          sequence prefix        sequence prefix features  pass   pol            made          gener.
-4.000000   4.000000          1        1.000000   [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]        0     0     0            1212               0
-
-finished run
-number of examples per pass = 1
-passes used = 1
-weighted example sum = 1
-weighted label sum = 0
-average loss = 4
-best constant = -inf
-total feature number = 0
diff --git a/test/train-sets/ref/sequencespan_data.nonldf.test-beam1.predict b/test/train-sets/ref/sequencespan_data.nonldf.test-beam1.predict
deleted file mode 100644
index 980567ab..00000000
--- a/test/train-sets/ref/sequencespan_data.nonldf.test-beam1.predict
+++ /dev/null
@@ -1 +0,0 @@
-2 1 1 2 2 1 6 7 7 7 7 1 6 4 1 
diff --git a/test/train-sets/ref/sequencespan_data.nonldf.test-beam1.stderr b/test/train-sets/ref/sequencespan_data.nonldf.test-beam1.stderr
deleted file mode 100644
index e7f9656c..00000000
--- a/test/train-sets/ref/sequencespan_data.nonldf.test-beam1.stderr
+++ /dev/null
@@ -1,24 +0,0 @@
-only testing
-Num weight bits = 18
-learning rate = 10
-initial_t = 1
-power_t = 0.5
-predictions = sequencespan_data.predict
-warning: setting searn_beam=1 is kind of a weird thing to do -- just don't use a beam at all
-using no cache
-Reading datafile = train-sets/sequencespan_data
-num sources = 1
-average    since         example     example  current  current  current
-loss       last          counter      weight    label  predict features
-average    since      sequence         example            current label      current predicted  current   cur   cur         predic.        examples
-loss       last        counter          weight          sequence prefix        sequence prefix features  pass   pol            made          gener.
-0.000000   0.000000          1        1.000000   [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]        0     0     0              71               0
-
-finished run
-number of examples per pass = 1
-passes used = 1
-weighted example sum = 1
-weighted label sum = 0
-average loss = 0
-best constant = -inf
-total feature number = 0
diff --git a/test/train-sets/ref/sequencespan_data.nonldf.test-beam20.stderr b/test/train-sets/ref/sequencespan_data.nonldf.test-beam20.stderr
deleted file mode 100644
index 3a2c0d6b..00000000
--- a/test/train-sets/ref/sequencespan_data.nonldf.test-beam20.stderr
+++ /dev/null
@@ -1 +0,0 @@
-switching to BILOU encoding for sequence span labeling
diff --git a/test/train-sets/ref/sequencespan_data.nonldf.train.stderr b/test/train-sets/ref/sequencespan_data.nonldf.train.stderr
index 1d7db07c..fcf72bcd 100644
--- a/test/train-sets/ref/sequencespan_data.nonldf.train.stderr
+++ b/test/train-sets/ref/sequencespan_data.nonldf.train.stderr
@@ -4,6 +4,7 @@ learning rate = 10
 initial_t = 1
 power_t = 0.5
 decay_learning_rate = 1
+no rollout!
 creating cache_file = train-sets/sequencespan_data.cache
 Reading datafile = train-sets/sequencespan_data
 num sources = 1
@@ -11,9 +12,9 @@ average    since      instance            current true      current predicted
 loss       last        counter           output prefix          output prefix  pass   pol     made     hits    gener  beta    
 6.000000   6.000000          1  [2 1 1 2 2 1 6 7 7 ..] [1 1 1 1 1 1 1 1 1 ..]     0     0       15        0       15  0.000000
 8.500000   11.000000         2  [2 1 1 2 2 1 6 7 7 ..] [2 1 6 4 1 6 4 1 6 ..]     1     0       30        0       30  0.000000
-6.000000   3.500000          4  [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]     3     0       60        0       60  0.000001
-3.000000   0.000000          8  [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]     7     0      120        0      120  0.000001
-1.500000   0.000000         16  [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]    15     0      240        0      240  0.000002
+6.000000   3.500000          4  [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]     3     0       60        0       60  0.000000
+3.000000   0.000000          8  [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]     7     0      120        0      120  0.000000
+1.500000   0.000000         16  [2 1 1 2 2 1 6 7 7 ..] [2 1 1 2 2 1 6 7 7 ..]    15     0      240        0      240  0.000000
 
 finished run
 number of examples per pass = 1
diff --git a/vowpalwabbit/main.cc b/vowpalwabbit/main.cc
index 30ecce6e..16f306cf 100644
--- a/vowpalwabbit/main.cc
+++ b/vowpalwabbit/main.cc
@@ -85,7 +85,7 @@ int main(int argc, char *argv[])
         }
         cerr << endl << "weighted example sum = " << all->sd->weighted_examples;
         cerr << endl << "weighted label sum = " << all->sd->weighted_labels;
-        if(all->holdout_set_off)
+        if(all->holdout_set_off || (all->sd->holdout_best_loss == FLT_MAX))
         {
             cerr << endl << "average loss = " << all->sd->sum_loss / all->sd->weighted_examples;
         }  
diff --git a/vowpalwabbit/search.cc b/vowpalwabbit/search.cc
index 8c0cec61..10382820 100644
--- a/vowpalwabbit/search.cc
+++ b/vowpalwabbit/search.cc
@@ -79,6 +79,7 @@ namespace Search {
     int mix_per_roll_policy;       // for MIX_PER_ROLL, we need to choose a policy to use; this is where it's stored (-2 means "not selected yet")
     bool no_caching;               // turn off caching
     size_t rollout_num_steps;      // how many calls of "loss" before we stop really predicting on rollouts and switch to oracle (0 means "infinite")
+    bool (*label_is_test)(void*);  // tell me if the label data from an example is test
     
     size_t t;                      // current search step
     size_t T;                      // length of root trajectory
@@ -191,7 +192,7 @@ namespace Search {
     else if (num_valid_policies == 2)
       pid = frand48() >= priv.beta;
     else {
-      // TODO: speed this up in the case that beta is small!
+      // SPEEDUP this up in the case that beta is small!
       float r = frand48();
       pid = 0;
 
@@ -236,11 +237,11 @@ namespace Search {
     // example because of off-by-ones
 
     if (PRINT_UPDATE_EVERY_EXAMPLE) return true;
-    if (PRINT_UPDATE_EVERY_PASS) return true;  // TODO: make this better
+    if (PRINT_UPDATE_EVERY_PASS) return true;  // SPEEDUP: make this better
     return (all.sd->weighted_examples + 1. >= all.sd->dump_interval) && !all.quiet && !all.bfgs;
   }
 
-  bool must_run_test(vw&all, vector<example*>ec) {
+  bool must_run_test(vw&all, vector<example*>ec, bool is_test_ex) {
     return
         (all.final_prediction_sink.size() > 0) ||   // if we have to produce output, we need to run this
         might_print_update(all) ||                  // if we have to print and update to stderr
@@ -251,6 +252,7 @@ namespace Search {
         //     OR holdout is off
         //     OR it's a test example
         ( //   (! all.quiet) &&  // had to disable this because of library mode!
+          (! is_test_ex) &&
           ( all.holdout_set_off ||                    // no holdout
             ec[0]->test_only ||
             (all.current_pass == 0)                   // we need error rates for progressive cost
@@ -305,7 +307,8 @@ namespace Search {
 
     float avg_loss = 0.;
     float avg_loss_since = 0.;
-    if (!all.holdout_set_off && all.current_pass >= 1) {
+    bool use_heldout_loss = (!all.holdout_set_off && all.current_pass >= 1) && (all.sd->weighted_holdout_examples > 0);
+    if (use_heldout_loss) {
       avg_loss       = safediv((float)all.sd->holdout_sum_loss, (float)all.sd->weighted_holdout_examples);
       avg_loss_since = safediv((float)all.sd->holdout_sum_loss_since_last_dump, (float)all.sd->weighted_holdout_examples_since_last_dump);
 
@@ -339,7 +342,7 @@ namespace Search {
       fprintf(stderr, " %15lusec", num_sec);
     }
 
-    if (!all.holdout_set_off && all.current_pass >= 1)
+    if (use_heldout_loss)
       fprintf(stderr, " h");
 
     fprintf(stderr, "\n");
@@ -471,7 +474,6 @@ namespace Search {
   }
 
   void search_declare_loss(search_private& priv, float loss) {
-    // TODO: if it's a test example...
     priv.loss_declared_cnt++;
     switch (priv.state) {
       case INIT_TEST:  priv.test_loss  += loss; break;
@@ -801,7 +803,7 @@ namespace Search {
   }
 
   void generate_training_example(search_private& priv, v_array<float>& losses, bool add_conditioning=true) {
-    // TODO: should we really subtract out min-loss?
+    // should we really subtract out min-loss?
     float min_loss = FLT_MAX, max_loss = -FLT_MAX;
     size_t num_min = 0;
     for (size_t i=0; i<losses.size(); i++) {
@@ -986,8 +988,6 @@ namespace Search {
             a = priv.is_ldf ? single_prediction_LDF(priv, ecs, ec_cnt, policy)
                             : single_prediction_notLDF(priv, *ecs, policy, allowed_actions, allowed_actions_cnt);
           
-          // TODO: need to look at oracle actions to make sure it's not "test example"!!!
-          
           if (gte_here) {
             cdbg << "INIT_TRAIN, NO_ROLLOUT, at least one oracle_actions" << endl;
             // we can generate a training example _NOW_ because we're not doing rollouts
@@ -1052,7 +1052,7 @@ namespace Search {
   }
   
   template <bool is_learn>
-  void train_single_example(search& sch) {
+  void train_single_example(search& sch, bool is_test_ex) {
     search_private& priv = *sch.priv;
     vw&all = *priv.all;
     bool ran_test = false;  // we must keep track so that even if we skip test, we still update # of examples seen
@@ -1060,7 +1060,7 @@ namespace Search {
 
     clear_cache_hash_map(priv);
     
-    if (must_run_test(all, priv.ec_seq)) {
+    if (must_run_test(all, priv.ec_seq, is_test_ex)) {
       cdbg << "======================================== INIT TEST (" << priv.current_policy << "," << priv.read_example_last_pass << ") ========================================" << endl;
 
       ran_test = true;
@@ -1073,19 +1073,21 @@ namespace Search {
       priv.task->run(sch, priv.ec_seq);
 
       // accumulate loss
-      if (priv.ec_seq[0]->test_only) {
-        all.sd->weighted_holdout_examples += 1.f;//test weight seen
-        all.sd->weighted_holdout_examples_since_last_dump += 1.f;
-        all.sd->weighted_holdout_examples_since_last_pass += 1.f;
-        all.sd->holdout_sum_loss += priv.test_loss;
-        all.sd->holdout_sum_loss_since_last_dump += priv.test_loss;
-        all.sd->holdout_sum_loss_since_last_pass += priv.test_loss;//since last pass
-      } else {
-        all.sd->weighted_examples += 1.f;
-        all.sd->total_features += priv.num_features;
-        all.sd->sum_loss += priv.test_loss;
-        all.sd->sum_loss_since_last_dump += priv.test_loss;
-        all.sd->example_number++;
+      if (! is_test_ex) { // we cannot accumulate loss on test examples!
+        if (priv.ec_seq[0]->test_only) {
+          all.sd->weighted_holdout_examples += 1.f;//test weight seen
+          all.sd->weighted_holdout_examples_since_last_dump += 1.f;
+          all.sd->weighted_holdout_examples_since_last_pass += 1.f;
+          all.sd->holdout_sum_loss += priv.test_loss;
+          all.sd->holdout_sum_loss_since_last_dump += priv.test_loss;
+          all.sd->holdout_sum_loss_since_last_pass += priv.test_loss;//since last pass
+        } else {
+          all.sd->weighted_examples += 1.f;
+          all.sd->total_features += priv.num_features;
+          all.sd->sum_loss += priv.test_loss;
+          all.sd->sum_loss_since_last_dump += priv.test_loss;
+          all.sd->example_number++;
+        }
       }
       
       // generate output
@@ -1097,7 +1099,7 @@ namespace Search {
     }
 
     // if we're not training, then we're done!
-    if ((!is_learn) || priv.ec_seq[0]->test_only || (!priv.all->training))
+    if ((!is_learn) || is_test_ex || priv.ec_seq[0]->test_only || (!priv.all->training))
       return;
 
     // SPEEDUP: if the oracle was never called, we can skip this!
@@ -1167,20 +1169,28 @@ namespace Search {
     if (priv.ec_seq.size() == 0)
       return;  // nothing to do :)
 
+    bool is_test_ex = false;
+    for (size_t i=0; i<priv.ec_seq.size(); i++)
+      if (priv.label_is_test(priv.ec_seq[i]->ld)) { is_test_ex = true; break; }
+    
     if (priv.task->run_setup) priv.task->run_setup(sch, priv.ec_seq);
     
     // if we're going to have to print to the screen, generate the "truth" string
     cdbg << "======================================== GET TRUTH STRING (" << priv.current_policy << "," << priv.read_example_last_pass << ") ========================================" << endl;
     if (might_print_update(all)) {
-      reset_search_structure(*sch.priv);
-      priv.state = GET_TRUTH_STRING;
-      priv.should_produce_string = true;
-      priv.truth_string->str("");
-      priv.task->run(sch, priv.ec_seq);
+      if (is_test_ex)
+        priv.truth_string->str("**test**");
+      else {
+        reset_search_structure(*sch.priv);
+        priv.state = GET_TRUTH_STRING;
+        priv.should_produce_string = true;
+        priv.truth_string->str("");
+        priv.task->run(sch, priv.ec_seq);
+      }
     }
 
     add_neighbor_features(priv);
-    train_single_example<is_learn>(sch);
+    train_single_example<is_learn>(sch, is_test_ex);
     del_neighbor_features(priv);
 
     if (priv.task->run_takedown) priv.task->run_takedown(sch, priv.ec_seq);
@@ -1260,6 +1270,8 @@ namespace Search {
       VW::cmd_string_replace_value(all->file_options,"--search_total_nb_policies", ss2.str());
     }
   }
+
+  bool mc_label_is_test(void* lab) { return MC::label_is_test((MC::multiclass*)lab); }
   
   void search_initialize(vw* all, search& sch) {
     search_private& priv = *sch.priv;
@@ -1270,6 +1282,8 @@ namespace Search {
     priv.examples_dont_change = false;
     priv.is_ldf = false;
 
+    priv.label_is_test = mc_label_is_test;
+    
     priv.A = 1;
     priv.num_learners = 1;
     priv.cb_learner = false;
@@ -1597,11 +1611,14 @@ namespace Search {
       throw exception();
     }
 
+    if (vm.count("search_rollout")) rollout_string = vm["search_rollout"].as<string>();
+    if (vm.count("search_rollin" )) rollin_string  = vm["search_rollin" ].as<string>();
+    
     if      (rollout_string.compare("policy") == 0)          priv.rollout_method = POLICY;
     else if (rollout_string.compare("oracle") == 0)          priv.rollout_method = ORACLE;
     else if (rollout_string.compare("mix_per_state") == 0)   priv.rollout_method = MIX_PER_STATE;
     else if (rollout_string.compare("mix_per_roll") == 0)    priv.rollout_method = MIX_PER_ROLL;
-    else if (rollout_string.compare("none") == 0)          { priv.rollout_method = NO_ROLLOUT; priv.no_caching = true; }
+    else if (rollout_string.compare("none") == 0)          { priv.rollout_method = NO_ROLLOUT; priv.no_caching = true; cerr << "no rollout!" << endl; }
     else {
       std::cerr << "error: --search_rollout must be 'policy', 'oracle', 'mix_per_state', 'mix_per_roll' or 'none'" << endl;
       throw exception();
@@ -1687,7 +1704,7 @@ namespace Search {
     }
     all.p->emptylines_separate_examples = true;
 
-    // default to OAA labels unless the task wants to override this!
+    // default to OAA labels unless the task wants to override this (which they can do in initialize)
     all.p->lp = MC::mc_label;
     if (priv.task)
       priv.task->initialize(*sch, priv.A, vm);
@@ -1770,6 +1787,15 @@ namespace Search {
     if ((opts & IS_LDF)                  != 0) this->priv->is_ldf = true;
   }
 
+  void search::set_label_parser(label_parser&lp, bool (*is_test)(void*)) {
+    if (this->priv->state != INITIALIZE) {
+      std::cerr << "error: task cannot set label parser except in initialize function!" << endl;
+      throw exception();
+    }
+    this->priv->all->p->lp = lp;
+    this->priv->label_is_test = is_test;
+  }
+  
   void search::set_num_learners(size_t num_learners) { this->priv->num_learners = num_learners; }
   void search::add_program_options(po::variables_map& vm, po::options_description& opts) { vm = add_options( *this->priv->all, opts ); }
 
diff --git a/vowpalwabbit/search.h b/vowpalwabbit/search.h
index c2482531..291ca672 100644
--- a/vowpalwabbit/search.h
+++ b/vowpalwabbit/search.h
@@ -32,6 +32,10 @@ namespace Search {
     // this should be an or ("|") of AUTO_CONDITION_FEATURES, etc.
     void set_options(uint32_t opts);
 
+    // change the default label parser, but you _must_ tell me how
+    // to detect test examples!
+    void set_label_parser(label_parser&lp, bool (*is_test)(void*));
+
     // for adding command-line options
     void add_program_options(po::variables_map& vw, po::options_description& opts);
     
diff --git a/vowpalwabbit/simple_label.cc b/vowpalwabbit/simple_label.cc
index 1777ac3d..7857ea98 100644
--- a/vowpalwabbit/simple_label.cc
+++ b/vowpalwabbit/simple_label.cc
@@ -216,6 +216,7 @@ bool summarize_holdout_set(vw& all, size_t& no_win_counter)
     return true;
   }
 
-  no_win_counter++;
+  if ((thisLoss != FLT_MAX) || (isfinite(all.sd->holdout_best_loss))) // it's only a loss if we're not infinite when the previous one wasn't infinite
+    no_win_counter++;
   return false;          
 }