SHOGUN  v1.1.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
VowpalWabbit.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2009 Yahoo! Inc. All rights reserved. The copyrights
3  * embodied in the content of this file are licensed under the BSD
4  * (revised) open source license.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * Written (W) 2011 Shashwat Lal Das
12  * Adaptation of Vowpal Wabbit v5.1.
13  * Copyright (C) 2011 Berlin Institute of Technology and Max-Planck-Society.
14  */
15 
17 
18 using namespace shogun;
19 
22 {
23  reg=NULL;
24  learner=NULL;
25  init(NULL);
26 }
27 
30 {
31  reg=NULL;
32  learner=NULL;
33  init(feat);
34 }
35 
37 {
38  SG_UNREF(env);
39  SG_UNREF(reg);
41 }
42 
44 {
45  if (reg->weight_vectors)
46  {
47  if (reg->weight_vectors[0])
50  }
51 
52  reg->init(env);
53  w = reg->weight_vectors[0];
54 }
55 
56 void CVowpalWabbit::set_adaptive(bool adaptive_learning)
57 {
58  if (adaptive_learning)
59  {
60  env->adaptive = true;
61  env->set_stride(2);
62  env->power_t = 0.;
64  }
65  else
66  env->adaptive = false;
67 }
68 
69 void CVowpalWabbit::set_exact_adaptive_norm(bool exact_adaptive)
70 {
71  if (exact_adaptive)
72  {
73  set_adaptive(true);
74  env->exact_adaptive_norm = true;
75  }
76  else
77  env->exact_adaptive_norm = false;
78 }
79 
80 void CVowpalWabbit::load_regressor(char* file_name)
81 {
82  reg->load_regressor(file_name);
83  w = reg->weight_vectors[0];
84  w_dim = 1 << env->num_bits;
85 }
86 
87 void CVowpalWabbit::set_regressor_out(char* file_name, bool is_text)
88 {
89  reg_name = file_name;
90  reg_dump_text = is_text;
91 }
92 
93 void CVowpalWabbit::set_prediction_out(char* file_name)
94 {
95  save_predictions = true;
96  prediction_fd = open(file_name, O_CREAT|O_TRUNC|O_WRONLY, 0666);
97  if (prediction_fd < 0)
98  SG_SERROR("Unable to open prediction file %s for writing!\n", file_name);
99 }
100 
102 {
103  env->pairs.push_back(pair);
104 }
105 
107 {
108  ASSERT(features || feat);
109  if (feat && (features != (CStreamingVwFeatures*) feat))
110  {
112  init((CStreamingVwFeatures*) feat);
113  }
114 
115  set_learner();
116 
117  VwExample* example = NULL;
118  vw_size_t current_pass = 0;
119 
120  const char* header_fmt = "%-10s %-10s %8s %8s %10s %8s %8s\n";
121 
122  if (!quiet)
123  {
124  SG_SPRINT(header_fmt,
125  "average", "since", "example", "example",
126  "current", "current", "current");
127  SG_SPRINT(header_fmt,
128  "loss", "last", "counter", "weight", "label", "predict", "features");
129  }
130 
132  while (env->passes_complete < env->num_passes)
133  {
134  while (features->get_next_example())
135  {
136  example = features->get_example();
137 
138  // Check if we shouldn't train (generally used for cache creation)
139  if (!no_training)
140  {
141  if (example->pass != current_pass)
142  {
143  env->eta *= env->eta_decay_rate;
144  current_pass = example->pass;
145  }
146 
147  predict_and_finalize(example);
148 
149  learner->train(example, example->eta_round);
150  example->eta_round = 0.;
151 
152  output_example(example);
153  }
154 
156  }
157  env->passes_complete++;
160  }
161  features->end_parser();
162 
163  if (env->l1_regularization > 0.)
164  {
165  uint32_t length = 1 << env->num_bits;
166  vw_size_t stride = env->stride;
168  for (uint32_t i = 0; i < length; i++)
169  reg->weight_vectors[0][stride*i] = real_weight(reg->weight_vectors[0][stride*i], gravity);
170  }
171 
172  if (reg_name != NULL)
173  reg->dump_regressor(reg_name, reg_dump_text);
174 
175  return true;
176 }
177 
179 {
180  float32_t prediction;
181  if (env->l1_regularization != 0.)
182  prediction = inline_l1_predict(ex);
183  else
184  prediction = inline_predict(ex);
185 
186  ex->final_prediction = 0;
187  ex->final_prediction += prediction;
188  ex->final_prediction = finalize_prediction(ex->final_prediction);
189  float32_t t = ex->example_t;
190 
191  if (ex->ld->label != FLT_MAX)
192  {
193  ex->loss = reg->get_loss(ex->final_prediction, ex->ld->label) * ex->ld->weight;
194  float64_t update = 0.;
196  {
197  float32_t sum_abs_x = 0.;
198  float32_t exact_norm = compute_exact_norm(ex, sum_abs_x);
199  update = (env->eta * exact_norm)/sum_abs_x;
200  env->update_sum += update;
201  ex->eta_round = reg->get_update(ex->final_prediction, ex->ld->label, update, exact_norm);
202  }
203  else
204  {
205  update = (env->eta)/pow(t, env->power_t) * ex->ld->weight;
206  ex->eta_round = reg->get_update(ex->final_prediction, ex->ld->label, update, ex->total_sum_feat_sq);
207  }
208  env->update_sum += update;
209  }
210 
211  return prediction;
212 }
213 
214 void CVowpalWabbit::init(CStreamingVwFeatures* feat)
215 {
216  features = feat;
217  env = feat->get_env();
218  reg = new CVwRegressor(env);
219  SG_REF(env);
220  SG_REF(reg);
221 
222  quiet = true;
223  no_training = false;
224  dump_interval = exp(1.);
225  sum_loss_since_last_dump = 0.;
226  reg_name = NULL;
227  reg_dump_text = true;
228  save_predictions = false;
229  prediction_fd = -1;
230 
231  w = reg->weight_vectors[0];
232  w_dim = 1 << env->num_bits;
233  bias = 0.;
234 }
235 
236 void CVowpalWabbit::set_learner()
237 {
238  if (env->adaptive)
240  else
242  SG_REF(learner);
243 }
244 
245 float32_t CVowpalWabbit::inline_l1_predict(VwExample* &ex)
246 {
247  vw_size_t thread_num = 0;
248 
249  float32_t prediction = ex->ld->get_initial();
250 
251  float32_t* weights = reg->weight_vectors[thread_num];
252  vw_size_t thread_mask = env->thread_mask;
253 
254  prediction += features->dense_dot_truncated(weights, ex, env->l1_regularization * env->update_sum);
255 
256  for (int32_t k = 0; k < env->pairs.get_num_elements(); k++)
257  {
258  char* i = env->pairs.get_element(k);
259 
260  v_array<VwFeature> temp = ex->atomics[(int32_t)(i[0])];
261  temp.begin = ex->atomics[(int32_t)(i[0])].begin;
262  temp.end = ex->atomics[(int32_t)(i[0])].end;
263  for (; temp.begin != temp.end; temp.begin++)
264  prediction += one_pf_quad_predict_trunc(weights, *temp.begin,
265  ex->atomics[(int32_t)(i[1])], thread_mask,
267  }
268 
269  return prediction;
270 }
271 
272 float32_t CVowpalWabbit::inline_predict(VwExample* &ex)
273 {
274  vw_size_t thread_num = 0;
275  float32_t prediction = ex->ld->initial;
276 
277  float32_t* weights = reg->weight_vectors[thread_num];
278  vw_size_t thread_mask = env->thread_mask;
279  prediction += features->dense_dot(weights, 0);
280 
281  for (int32_t k = 0; k < env->pairs.get_num_elements(); k++)
282  {
283  char* i = env->pairs.get_element(k);
284 
285  v_array<VwFeature> temp = ex->atomics[(int32_t)(i[0])];
286  temp.begin = ex->atomics[(int32_t)(i[0])].begin;
287  temp.end = ex->atomics[(int32_t)(i[0])].end;
288  for (; temp.begin != temp.end; temp.begin++)
289  prediction += one_pf_quad_predict(weights, *temp.begin,
290  ex->atomics[(int32_t)(i[1])],
291  thread_mask);
292  }
293 
294  return prediction;
295 }
296 
297 float32_t CVowpalWabbit::finalize_prediction(float32_t ret)
298 {
299  if (isnan(ret))
300  return 0.5;
301  if (ret > env->max_label)
302  return env->max_label;
303  if (ret < env->min_label)
304  return env->min_label;
305 
306  return ret;
307 }
308 
309 void CVowpalWabbit::output_example(VwExample* &example)
310 {
311  if (!quiet)
312  {
313  sum_loss_since_last_dump += example->loss;
314  if (env->weighted_examples + example->ld->weight > dump_interval)
315  {
316  print_update(example);
317  dump_interval *= 2;
318  }
319  }
320 
321  if (save_predictions)
322  {
323  float32_t wt = 0.;
324  if (reg->weight_vectors)
325  wt = reg->weight_vectors[0][0];
326 
327  output_prediction(prediction_fd, example->final_prediction, wt * example->global_weight, example->tag);
328  }
329 }
330 
331 void CVowpalWabbit::print_update(VwExample* &ex)
332 {
333  SG_SPRINT("%-10.6f %-10.6f %8lld %8.1f %8.4f %8.4f %8lu\n",
334  (env->sum_loss + ex->loss)/(env->weighted_examples + ex->ld->weight),
335  sum_loss_since_last_dump/(env->weighted_examples + ex->ld->weight - old_weighted_examples),
336  env->example_number + 1,
337  env->weighted_examples + ex->ld->weight,
338  ex->ld->label,
339  ex->final_prediction,
340  (long unsigned int)ex->num_features);
341  sum_loss_since_last_dump = 0.0;
342  old_weighted_examples = env->weighted_examples + ex->ld->weight;
343 }
344 
345 
346 void CVowpalWabbit::output_prediction(int32_t f, float32_t res, float32_t weight, v_array<char> tag)
347 {
348  if (f >= 0)
349  {
350  char temp[30];
351  int32_t num = sprintf(temp, "%f", res);
352  ssize_t t;
353  t = write(f, temp, num);
354  if (t != num)
355  SG_SERROR("Write error!\n");
356 
357  if (tag.begin != tag.end)
358  {
359  temp[0] = ' ';
360  t = write(f, temp, 1);
361  if (t != 1)
362  SG_SERROR("Write error!\n");
363 
364  t = write(f, tag.begin, sizeof(char)*tag.index());
365  if (t != (ssize_t) (sizeof(char)*tag.index()))
366  SG_SERROR("Write error!\n");
367  }
368 
369  temp[0] = '\n';
370  t = write(f, temp, 1);
371  if (t != 1)
372  SG_SERROR("Write error!\n");
373  }
374 }
375 
376 void CVowpalWabbit::set_verbose(bool verbose)
377 {
378  quiet=verbose==false;
379 }
380 
381 
383 {
384  // We must traverse the features in _precisely_ the same order as during training.
385  vw_size_t thread_mask = env->thread_mask;
386  vw_size_t thread_num = 0;
387 
389  if (g == 0) return 0.;
390 
391  float32_t xGx = 0.;
392 
393  float32_t* weights = reg->weight_vectors[thread_num];
394  for (vw_size_t* i = ex->indices.begin; i != ex->indices.end; i++)
395  {
396  for (VwFeature* f = ex->atomics[*i].begin; f != ex->atomics[*i].end; f++)
397  {
398  float32_t* w_vec = &weights[f->weight_index & thread_mask];
399  float32_t t = f->x * CMath::invsqrt(w_vec[1] + g * f->x * f->x);
400  xGx += t * f->x;
401  sum_abs_x += fabsf(f->x);
402  }
403  }
404 
405  for (int32_t k = 0; k < env->pairs.get_num_elements(); k++)
406  {
407  char* i = env->pairs.get_element(k);
408 
409  v_array<VwFeature> temp = ex->atomics[(int32_t)(i[0])];
410  temp.begin = ex->atomics[(int32_t)(i[0])].begin;
411  temp.end = ex->atomics[(int32_t)(i[0])].end;
412  for (; temp.begin != temp.end; temp.begin++)
413  xGx += compute_exact_norm_quad(weights, *temp.begin, ex->atomics[(int32_t)(i[1])], thread_mask, g, sum_abs_x);
414  }
415 
416  return xGx;
417 }
418 
420  vw_size_t mask, float32_t g, float32_t& sum_abs_x)
421 {
422  vw_size_t halfhash = quadratic_constant * page_feature.weight_index;
423  float32_t xGx = 0.;
424  float32_t update2 = g * page_feature.x * page_feature.x;
425  for (VwFeature* elem = offer_features.begin; elem != offer_features.end; elem++)
426  {
427  float32_t* w_vec = &weights[(halfhash + elem->weight_index) & mask];
428  float32_t t = elem->x * CMath::invsqrt(w_vec[1] + update2 * elem->x * elem->x);
429  xGx += t * elem->x;
430  sum_abs_x += fabsf(elem->x);
431  }
432  return xGx;
433 }

SHOGUN Machine Learning Toolbox - Documentation