fann/src/fann_cascade.c

   1 /*
   2   Fast Artificial Neural Network Library (fann)
   3   Copyright (C) 2003 Steffen Nissen (lukesky@diku.dk)
   4
   5   This library is free software; you can redistribute it and/or
   6   modify it under the terms of the GNU Lesser General Public
   7   License as published by the Free Software Foundation; either
   8   version 2.1 of the License, or (at your option) any later version.
   9
  10   This library is distributed in the hope that it will be useful,
  11   but WITHOUT ANY WARRANTY; without even the implied warranty of
  12   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13   Lesser General Public License for more details.
  14
  15   You should have received a copy of the GNU Lesser General Public
  16   License along with this library; if not, write to the Free Software
  17   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18 */
  19
  20 #include "config.h"
  21 #include "fann.h"
  22 #include "string.h"
  23
  24 #ifndef FIXEDFANN
  25
  26 /* #define CASCADE_DEBUG */
  27 /* #define CASCADE_DEBUG_FULL */
  28
  29 void fann_print_connections_raw(struct fann *ann)
  30 {
  31         unsigned int i;
  32
  33         for(i = 0; i < ann->total_connections_allocated; i++)
  34         {
  35                 if(i == ann->total_connections)
  36                 {
  37                         printf("* ");
  38                 }
  39                 printf("%f ", ann->weights[i]);
  40         }
  41         printf("\n\n");
  42 }
  43
  44 /* Cascade training directly on the training data.
  45    The connected_neurons pointers are not valid during training,
  46    but they will be again after training.
  47  */
  48 FANN_EXTERNAL void FANN_API fann_cascadetrain_on_data(struct fann *ann, struct fann_train_data *data,
  49                                                                                 unsigned int max_neurons,
  50                                                                                 unsigned int neurons_between_reports,
  51                                                                                 float desired_error)
  52 {
  53         float error;
  54         unsigned int i;
  55         unsigned int total_epochs = 0;
  56         int desired_error_reached;
  57
  58         if(neurons_between_reports && ann->callback == NULL)
  59         {
  60                 printf("Max neurons %3d. Desired error: %.6f\n", max_neurons, desired_error);
  61         }
  62
  63         for(i = 1; i <= max_neurons; i++)
  64         {
  65                 /* train output neurons */
  66                 total_epochs += fann_train_outputs(ann, data, desired_error);
  67                 error = fann_get_MSE(ann);
  68                 desired_error_reached = fann_desired_error_reached(ann, desired_error);
  69
  70                 /* print current error */
  71                 if(neurons_between_reports &&
  72                    (i % neurons_between_reports == 0
  73                         || i == max_neurons || i == 1 || desired_error_reached == 0))
  74                 {
  75                         if(ann->callback == NULL)
  76                         {
  77                                 printf
  78                                         ("Neurons     %3d. Current error: %.6f. Total error:%8.4f. Epochs %5d. Bit fail %3d",
  79                                          i, error, ann->MSE_value, total_epochs, ann->num_bit_fail);
  80                                 if((ann->last_layer-2) != ann->first_layer)
  81                                 {
  82                                         printf(". candidate steepness %.2f. function %s",
  83                                            (ann->last_layer-2)->first_neuron->activation_steepness,
  84                                            FANN_ACTIVATIONFUNC_NAMES[(ann->last_layer-2)->first_neuron->activation_function]);
  85                                 }
  86                                 printf("\n");
  87                         }
  88                         else if((*ann->callback) (ann, data, max_neurons,
  89                                 neurons_between_reports, desired_error, total_epochs) == -1)
  90                         {
  91                                 /* you can break the training by returning -1 */
  92                                 break;
  93                         }
  94                 }
  95
  96                 if(desired_error_reached == 0)
  97                         break;
  98
  99                 if(fann_initialize_candidates(ann) == -1)
 100                 {
 101                         /* Unable to initialize room for candidates */
 102                         break;
 103                 }
 104
 105                 /* train new candidates */
 106                 total_epochs += fann_train_candidates(ann, data);
 107
 108                 /* this installs the best candidate */
 109                 fann_install_candidate(ann);
 110         }
 111
 112         /* Train outputs one last time but without any desired error */
 113         total_epochs += fann_train_outputs(ann, data, 0.0);
 114
 115         if(neurons_between_reports && ann->callback == NULL)
 116         {
 117                 printf("Train outputs    Current error: %.6f. Epochs %6d\n", fann_get_MSE(ann),
 118                            total_epochs);
 119         }
 120
 121         /* Set pointers in connected_neurons
 122          * This is ONLY done in the end of cascade training,
 123          * since there is no need for them during training.
 124          */
 125         fann_set_shortcut_connections(ann);
 126 }
 127
 128 FANN_EXTERNAL void FANN_API fann_cascadetrain_on_file(struct fann *ann, const char *filename,
 129                                                                                                           unsigned int max_neurons,
 130                                                                                                           unsigned int neurons_between_reports,
 131                                                                                                           float desired_error)
 132 {
 133         struct fann_train_data *data = fann_read_train_from_file(filename);
 134
 135         if(data == NULL)
 136         {
 137                 return;
 138         }
 139         fann_cascadetrain_on_data(ann, data, max_neurons, neurons_between_reports, desired_error);
 140         fann_destroy_train(data);
 141 }
 142
 143 int fann_train_outputs(struct fann *ann, struct fann_train_data *data, float desired_error)
 144 {
 145         float error, initial_error, error_improvement;
 146         float target_improvement = 0.0;
 147         float backslide_improvement = -1.0e20f;
 148         unsigned int i;
 149         unsigned int max_epochs = ann->cascade_max_out_epochs;
 150         unsigned int stagnation = max_epochs;
 151
 152         /* TODO should perhaps not clear all arrays */
 153         fann_clear_train_arrays(ann);
 154
 155         /* run an initial epoch to set the initital error */
 156         initial_error = fann_train_outputs_epoch(ann, data);
 157
 158         if(fann_desired_error_reached(ann, desired_error) == 0)
 159                 return 1;
 160
 161         for(i = 1; i < max_epochs; i++)
 162         {
 163                 error = fann_train_outputs_epoch(ann, data);
 164
 165                 /*printf("Epoch %6d. Current error: %.6f. Bit fail %d.\n", i, error, ann->num_bit_fail); */
 166
 167                 if(fann_desired_error_reached(ann, desired_error) == 0)
 168                 {
 169 #ifdef CASCADE_DEBUG
 170                         printf("Error %f < %f\n", error, desired_error);
 171 #endif
 172                         return i + 1;
 173                 }
 174
 175                 /* Improvement since start of train */
 176                 error_improvement = initial_error - error;
 177
 178                 /* After any significant change, set a new goal and
 179                  * allow a new quota of epochs to reach it */
 180                 if((error_improvement > target_improvement) || (error_improvement < backslide_improvement))
 181                 {
 182                         /*printf("error_improvement=%f, target_improvement=%f, backslide_improvement=%f, stagnation=%d\n", error_improvement, target_improvement, backslide_improvement, stagnation); */
 183
 184                         target_improvement = error_improvement * (1.0f + ann->cascade_output_change_fraction);
 185                         backslide_improvement = error_improvement * (1.0f - ann->cascade_output_change_fraction);
 186                         stagnation = i + ann->cascade_output_stagnation_epochs;
 187                 }
 188
 189                 /* No improvement in allotted period, so quit */
 190                 if(i >= stagnation)
 191                 {
 192                         return i + 1;
 193                 }
 194         }
 195
 196         return max_epochs;
 197 }
 198
 199 float fann_train_outputs_epoch(struct fann *ann, struct fann_train_data *data)
 200 {
 201         unsigned int i;
 202
 203         fann_reset_MSE(ann);
 204
 205         for(i = 0; i < data->num_data; i++)
 206         {
 207                 fann_run(ann, data->input[i]);
 208                 fann_compute_MSE(ann, data->output[i]);
 209                 fann_update_slopes_batch(ann, ann->last_layer - 1, ann->last_layer - 1);
 210         }
 211
 212         switch (ann->training_algorithm)
 213         {
 214                 case FANN_TRAIN_RPROP:
 215                         fann_update_weights_irpropm(ann, (ann->last_layer - 1)->first_neuron->first_con,
 216                                                                                 ann->total_connections);
 217                         break;
 218                 case FANN_TRAIN_QUICKPROP:
 219                         fann_update_weights_quickprop(ann, data->num_data,
 220                                                                                   (ann->last_layer - 1)->first_neuron->first_con,
 221                                                                                   ann->total_connections);
 222                         break;
 223                 case FANN_TRAIN_BATCH:
 224                 case FANN_TRAIN_INCREMENTAL:
 225                         fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
 226         }
 227
 228         return fann_get_MSE(ann);
 229 }
 230
 231 int fann_reallocate_connections(struct fann *ann, unsigned int total_connections)
 232 {
 233         /* The connections are allocated, but the pointers inside are
 234          * first moved in the end of the cascade training session.
 235          */
 236
 237 #ifdef CASCADE_DEBUG
 238         printf("realloc from %d to %d\n", ann->total_connections_allocated, total_connections);
 239 #endif
 240         ann->connections =
 241                 (struct fann_neuron **) realloc(ann->connections,
 242                                                                                 total_connections * sizeof(struct fann_neuron *));
 243         if(ann->connections == NULL)
 244         {
 245                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 246                 return -1;
 247         }
 248
 249         ann->weights = (fann_type *) realloc(ann->weights, total_connections * sizeof(fann_type));
 250         if(ann->weights == NULL)
 251         {
 252                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 253                 return -1;
 254         }
 255
 256         ann->train_slopes =
 257                 (fann_type *) realloc(ann->train_slopes, total_connections * sizeof(fann_type));
 258         if(ann->train_slopes == NULL)
 259         {
 260                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 261                 return -1;
 262         }
 263
 264         ann->prev_steps = (fann_type *) realloc(ann->prev_steps, total_connections * sizeof(fann_type));
 265         if(ann->prev_steps == NULL)
 266         {
 267                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 268                 return -1;
 269         }
 270
 271         ann->prev_train_slopes =
 272                 (fann_type *) realloc(ann->prev_train_slopes, total_connections * sizeof(fann_type));
 273         if(ann->prev_train_slopes == NULL)
 274         {
 275                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 276                 return -1;
 277         }
 278
 279         ann->total_connections_allocated = total_connections;
 280
 281         return 0;
 282 }
 283
 284 int fann_reallocate_neurons(struct fann *ann, unsigned int total_neurons)
 285 {
 286         struct fann_layer *layer_it;
 287         struct fann_neuron *neurons;
 288         unsigned int num_neurons = 0;
 289         unsigned int num_neurons_so_far = 0;
 290
 291         neurons =
 292                 (struct fann_neuron *) realloc(ann->first_layer->first_neuron,
 293                                                                            total_neurons * sizeof(struct fann_neuron));
 294         ann->total_neurons_allocated = total_neurons;
 295
 296         if(neurons == NULL)
 297         {
 298                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 299                 return -1;
 300         }
 301
 302         /* Also allocate room for more train_errors */
 303         ann->train_errors = (fann_type *) realloc(ann->train_errors, total_neurons * sizeof(fann_type));
 304         if(ann->train_errors == NULL)
 305         {
 306                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 307                 return -1;
 308         }
 309
 310         if(neurons != ann->first_layer->first_neuron)
 311         {
 312                 /* Then the memory has moved, also move the pointers */
 313
 314 #ifdef CASCADE_DEBUG_FULL
 315                 printf("Moving neuron pointers\n");
 316 #endif
 317
 318                 /* Move pointers from layers to neurons */
 319                 for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++)
 320                 {
 321                         num_neurons = layer_it->last_neuron - layer_it->first_neuron;
 322                         layer_it->first_neuron = neurons + num_neurons_so_far;
 323                         layer_it->last_neuron = layer_it->first_neuron + num_neurons;
 324                         num_neurons_so_far += num_neurons;
 325                 }
 326         }
 327
 328         return 0;
 329 }
 330
 331 int fann_initialize_candidates(struct fann *ann)
 332 {
 333         /* The candidates are allocated after the normal neurons and connections,
 334          * but there is an empty place between the real neurons and the candidate neurons,
 335          * so that it will be possible to make room when the chosen candidate are copied in
 336          * on the desired place.
 337          */
 338         unsigned int neurons_to_allocate, connections_to_allocate;
 339         unsigned int num_candidates = fann_get_cascade_num_candidates(ann);
 340         unsigned int num_neurons = ann->total_neurons + num_candidates + 1;
 341         unsigned int candidate_connections_in = ann->total_neurons - ann->num_output;
 342         unsigned int candidate_connections_out = ann->num_output;
 343
 344         /* the number of connections going into a and out of a candidate is
 345          * ann->total_neurons */
 346         unsigned int num_connections =
 347                 ann->total_connections + (ann->total_neurons * (num_candidates + 1));
 348         unsigned int first_candidate_connection = ann->total_connections + ann->total_neurons;
 349         unsigned int first_candidate_neuron = ann->total_neurons + 1;
 350         unsigned int connection_it, i, j, k, candidate_index;
 351         struct fann_neuron *neurons;
 352         fann_type initial_slope;
 353
 354         /* First make sure that there is enough room, and if not then allocate a
 355          * bit more so that we do not need to allocate more room each time.
 356          */
 357         if(num_neurons > ann->total_neurons_allocated)
 358         {
 359                 /* Then we need to allocate more neurons
 360                  * Allocate half as many neurons as already exist (at least ten)
 361                  */
 362                 neurons_to_allocate = num_neurons + num_neurons / 2;
 363                 if(neurons_to_allocate < num_neurons + 10)
 364                 {
 365                         neurons_to_allocate = num_neurons + 10;
 366                 }
 367
 368                 if(fann_reallocate_neurons(ann, neurons_to_allocate) == -1)
 369                 {
 370                         return -1;
 371                 }
 372         }
 373
 374         if(num_connections > ann->total_connections_allocated)
 375         {
 376                 /* Then we need to allocate more connections
 377                  * Allocate half as many connections as already exist
 378                  * (at least enough for ten neurons)
 379                  */
 380                 connections_to_allocate = num_connections + num_connections / 2;
 381                 if(connections_to_allocate < num_connections + ann->total_neurons * 10)
 382                 {
 383                         connections_to_allocate = num_connections + ann->total_neurons * 10;
 384                 }
 385
 386                 if(fann_reallocate_connections(ann, connections_to_allocate) == -1)
 387                 {
 388                         return -1;
 389                 }
 390         }
 391
 392         /* Set the neurons.
 393          */
 394         connection_it = first_candidate_connection;
 395         neurons = ann->first_layer->first_neuron;
 396         candidate_index = first_candidate_neuron;
 397
 398         for(i = 0; i < ann->cascade_activation_functions_count; i++)
 399         {
 400                 for(j = 0; j < ann->cascade_activation_steepnesses_count; j++)
 401                 {
 402                         for(k = 0; k < ann->cascade_num_candidate_groups; k++)
 403                         {
 404                                 /* TODO candidates should actually be created both in
 405                                  * the last layer before the output layer, and in a new layer.
 406                                  */
 407                                 neurons[candidate_index].value = 0;
 408                                 neurons[candidate_index].sum = 0;
 409
 410                                 neurons[candidate_index].activation_function =
 411                                         ann->cascade_activation_functions[i];
 412                                 neurons[candidate_index].activation_steepness =
 413                                         ann->cascade_activation_steepnesses[j];
 414
 415                                 neurons[candidate_index].first_con = connection_it;
 416                                 connection_it += candidate_connections_in;
 417                                 neurons[candidate_index].last_con = connection_it;
 418                                 /* We have no specific pointers to the output weights, but they are
 419                                  * available after last_con */
 420                                 connection_it += candidate_connections_out;
 421                                 ann->train_errors[candidate_index] = 0;
 422                                 candidate_index++;
 423                         }
 424                 }
 425         }
 426
 427         /* Now randomize the weights and zero out the arrays that needs zeroing out.
 428          */
 429 #ifdef CASCADE_DEBUG_FULL
 430         printf("random cand weight [%d ... %d]\n", first_candidate_connection, num_connections - 1);
 431 #endif
 432         if(ann->training_algorithm == FANN_TRAIN_RPROP)
 433         {
 434                 initial_slope = ann->rprop_delta_zero;
 435         }
 436         else
 437         {
 438                 initial_slope = 0.0;
 439         }
 440         for(i = first_candidate_connection; i < num_connections; i++)
 441         {
 442                 ann->weights[i] = fann_random_weight();
 443                 /*ann->weights[i] = fann_rand(-0.25,0.25);*/
 444                 ann->train_slopes[i] = 0;
 445                 ann->prev_steps[i] = 0;
 446                 ann->prev_train_slopes[i] = initial_slope;
 447         }
 448
 449         return 0;
 450 }
 451
 452 int fann_train_candidates(struct fann *ann, struct fann_train_data *data)
 453 {
 454         fann_type best_cand_score = 0.0;
 455         fann_type target_cand_score = 0.0;
 456         fann_type backslide_cand_score = -1.0e20f;
 457         unsigned int i;
 458         unsigned int max_epochs = ann->cascade_max_cand_epochs;
 459         unsigned int stagnation = max_epochs;
 460
 461         if(ann->cascade_candidate_scores == NULL)
 462         {
 463                 ann->cascade_candidate_scores =
 464                         (fann_type *) malloc(fann_get_cascade_num_candidates(ann) * sizeof(fann_type));
 465                 if(ann->cascade_candidate_scores == NULL)
 466                 {
 467                         fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 468                         return 0;
 469                 }
 470         }
 471
 472         for(i = 0; i < max_epochs; i++)
 473         {
 474                 best_cand_score = fann_train_candidates_epoch(ann, data);
 475
 476                 if(best_cand_score / ann->MSE_value > ann->cascade_candidate_limit)
 477                 {
 478 #ifdef CASCADE_DEBUG
 479                         printf("above candidate limit %f/%f > %f", best_cand_score, ann->MSE_value,
 480                                    ann->cascade_candidate_limit);
 481 #endif
 482                         return i + 1;
 483                 }
 484
 485                 if((best_cand_score > target_cand_score) || (best_cand_score < backslide_cand_score))
 486                 {
 487 #ifdef CASCADE_DEBUG_FULL
 488                         printf("Best candidate score %f, real score: %f\n", ann->MSE_value - best_cand_score,
 489                                    best_cand_score);
 490                         /* printf("best_cand_score=%f, target_cand_score=%f, backslide_cand_score=%f, stagnation=%d\n", best_cand_score, target_cand_score, backslide_cand_score, stagnation); */
 491 #endif
 492
 493                         target_cand_score = best_cand_score * (1.0f + ann->cascade_candidate_change_fraction);
 494                         backslide_cand_score = best_cand_score * (1.0f - ann->cascade_candidate_change_fraction);
 495                         stagnation = i + ann->cascade_candidate_stagnation_epochs;
 496                 }
 497
 498                 /* No improvement in allotted period, so quit */
 499                 if(i >= stagnation)
 500                 {
 501 #ifdef CASCADE_DEBUG
 502                         printf("Stagnation with %d epochs, best candidate score %f, real score: %f\n", i + 1,
 503                                    ann->MSE_value - best_cand_score, best_cand_score);
 504 #endif
 505                         return i + 1;
 506                 }
 507         }
 508
 509 #ifdef CASCADE_DEBUG
 510         printf("Max epochs %d reached, best candidate score %f, real score: %f\n", max_epochs,
 511                    ann->MSE_value - best_cand_score, best_cand_score);
 512 #endif
 513         return max_epochs;
 514 }
 515
 516 void fann_update_candidate_slopes(struct fann *ann)
 517 {
 518         struct fann_neuron *neurons = ann->first_layer->first_neuron;
 519         struct fann_neuron *first_cand = neurons + ann->total_neurons + 1;
 520         struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann);
 521         struct fann_neuron *cand_it;
 522         unsigned int i, j, num_connections;
 523         unsigned int num_output = ann->num_output;
 524         fann_type max_sum, cand_sum, activation, derived, error_value, diff, cand_score;
 525         fann_type *weights, *cand_out_weights, *cand_slopes, *cand_out_slopes;
 526         fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
 527
 528         for(cand_it = first_cand; cand_it < last_cand; cand_it++)
 529         {
 530                 cand_score = ann->cascade_candidate_scores[cand_it - first_cand];
 531                 error_value = 0.0;
 532
 533                 /* code more or less stolen from fann_run to fast forward pass
 534                  */
 535                 cand_sum = 0.0;
 536                 num_connections = cand_it->last_con - cand_it->first_con;
 537                 weights = ann->weights + cand_it->first_con;
 538
 539                 /* unrolled loop start */
 540                 i = num_connections & 3;        /* same as modulo 4 */
 541                 switch (i)
 542                 {
 543                         case 3:
 544                                 cand_sum += weights[2] * neurons[2].value;
 545                         case 2:
 546                                 cand_sum += weights[1] * neurons[1].value;
 547                         case 1:
 548                                 cand_sum += weights[0] * neurons[0].value;
 549                         case 0:
 550                                 break;
 551                 }
 552
 553                 for(; i != num_connections; i += 4)
 554                 {
 555                         cand_sum +=
 556                                 weights[i] * neurons[i].value +
 557                                 weights[i + 1] * neurons[i + 1].value +
 558                                 weights[i + 2] * neurons[i + 2].value + weights[i + 3] * neurons[i + 3].value;
 559                 }
 560                 /*
 561                  * for(i = 0; i < num_connections; i++){
 562                  * cand_sum += weights[i] * neurons[i].value;
 563                  * }
 564                  */
 565                 /* unrolled loop end */
 566
 567                 max_sum = 150/cand_it->activation_steepness;
 568                 if(cand_sum > max_sum)
 569                         cand_sum = max_sum;
 570                 else if(cand_sum < -max_sum)
 571                         cand_sum = -max_sum;
 572
 573                 activation =
 574                         fann_activation(ann, cand_it->activation_function, cand_it->activation_steepness,
 575                                                         cand_sum);
 576                 /* printf("%f = sigmoid(%f);\n", activation, cand_sum); */
 577
 578                 cand_it->sum = cand_sum;
 579                 cand_it->value = activation;
 580
 581                 derived = fann_activation_derived(cand_it->activation_function,
 582                                                                                   cand_it->activation_steepness, activation, cand_sum);
 583
 584                 /* The output weights is located right after the input weights in
 585                  * the weight array.
 586                  */
 587                 cand_out_weights = weights + num_connections;
 588
 589                 cand_out_slopes = ann->train_slopes + cand_it->first_con + num_connections;
 590                 for(j = 0; j < num_output; j++)
 591                 {
 592                         diff = (activation * cand_out_weights[j]) - output_train_errors[j];
 593 #ifdef CASCADE_DEBUG_FULL
 594                         /* printf("diff = %f = (%f * %f) - %f;\n", diff, activation, cand_out_weights[j], output_train_errors[j]); */
 595 #endif
 596                         cand_out_slopes[j] -= 2.0f * diff * activation;
 597 #ifdef CASCADE_DEBUG_FULL
 598                         /* printf("cand_out_slopes[%d] <= %f += %f * %f;\n", j, cand_out_slopes[j], diff, activation); */
 599 #endif
 600                         error_value += diff * cand_out_weights[j];
 601                         cand_score -= (diff * diff);
 602 #ifdef CASCADE_DEBUG_FULL
 603                         /* printf("cand_score[%d][%d] = %f -= (%f * %f)\n", cand_it - first_cand, j, cand_score, diff, diff); */
 604
 605                         printf("cand[%d]: error=%f, activation=%f, diff=%f, slope=%f\n", cand_it - first_cand,
 606                                    output_train_errors[j], (activation * cand_out_weights[j]), diff,
 607                                    -2.0 * diff * activation);
 608 #endif
 609                 }
 610
 611                 ann->cascade_candidate_scores[cand_it - first_cand] = cand_score;
 612                 error_value *= derived;
 613
 614                 cand_slopes = ann->train_slopes + cand_it->first_con;
 615                 for(i = 0; i < num_connections; i++)
 616                 {
 617                         cand_slopes[i] -= error_value * neurons[i].value;
 618                 }
 619         }
 620 }
 621
 622 void fann_update_candidate_weights(struct fann *ann, unsigned int num_data)
 623 {
 624         struct fann_neuron *first_cand = (ann->last_layer - 1)->last_neuron + 1;        /* there is an empty neuron between the actual neurons and the candidate neuron */
 625         struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann) - 1;
 626
 627         switch (ann->training_algorithm)
 628         {
 629                 case FANN_TRAIN_RPROP:
 630                         fann_update_weights_irpropm(ann, first_cand->first_con,
 631                                                                                 last_cand->last_con + ann->num_output);
 632                         break;
 633                 case FANN_TRAIN_QUICKPROP:
 634                         fann_update_weights_quickprop(ann, num_data, first_cand->first_con,
 635                                                                                   last_cand->last_con + ann->num_output);
 636                         break;
 637                 case FANN_TRAIN_BATCH:
 638                 case FANN_TRAIN_INCREMENTAL:
 639                         fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
 640                         break;
 641         }
 642 }
 643
 644 fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data *data)
 645 {
 646         unsigned int i, j;
 647         unsigned int best_candidate;
 648         fann_type best_score;
 649         unsigned int num_cand = fann_get_cascade_num_candidates(ann);
 650         fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
 651         struct fann_neuron *output_neurons = (ann->last_layer - 1)->first_neuron;
 652
 653         for(i = 0; i < num_cand; i++)
 654         {
 655                 /* The ann->MSE_value is actually the sum squared error */
 656                 ann->cascade_candidate_scores[i] = ann->MSE_value;
 657         }
 658         /*printf("start score: %f\n", ann->MSE_value); */
 659
 660         for(i = 0; i < data->num_data; i++)
 661         {
 662                 fann_run(ann, data->input[i]);
 663
 664                 for(j = 0; j < ann->num_output; j++)
 665                 {
 666                         /* TODO only debug, but the error is in opposite direction, this might be usefull info */
 667                         /*          if(output_train_errors[j] != (ann->output[j] - data->output[i][j])){
 668                          * printf("difference in calculated error at %f != %f; %f = %f - %f;\n", output_train_errors[j], (ann->output[j] - data->output[i][j]), output_train_errors[j], ann->output[j], data->output[i][j]);
 669                          * } */
 670
 671                         /*
 672                          * output_train_errors[j] = (data->output[i][j] - ann->output[j])/2;
 673                          * output_train_errors[j] = ann->output[j] - data->output[i][j];
 674                          */
 675
 676                         output_train_errors[j] = (data->output[i][j] - ann->output[j]);
 677
 678                         switch (output_neurons[j].activation_function)
 679                         {
 680                                 case FANN_LINEAR_PIECE_SYMMETRIC:
 681                                 case FANN_SIGMOID_SYMMETRIC:
 682                                 case FANN_SIGMOID_SYMMETRIC_STEPWISE:
 683                                 case FANN_THRESHOLD_SYMMETRIC:
 684                                 case FANN_ELLIOT_SYMMETRIC:
 685                                 case FANN_GAUSSIAN_SYMMETRIC:
 686                                         output_train_errors[j] /= 2.0;
 687                                         break;
 688                                 case FANN_LINEAR:
 689                                 case FANN_THRESHOLD:
 690                                 case FANN_SIGMOID:
 691                                 case FANN_SIGMOID_STEPWISE:
 692                                 case FANN_GAUSSIAN:
 693                                 case FANN_GAUSSIAN_STEPWISE:
 694                                 case FANN_ELLIOT:
 695                                 case FANN_LINEAR_PIECE:
 696                                         break;
 697                         }
 698                 }
 699
 700                 fann_update_candidate_slopes(ann);
 701         }
 702
 703         fann_update_candidate_weights(ann, data->num_data);
 704
 705         /* find the best candidate score */
 706         best_candidate = 0;
 707         best_score = ann->cascade_candidate_scores[best_candidate];
 708         for(i = 1; i < num_cand; i++)
 709         {
 710                 /*struct fann_neuron *cand = ann->first_layer->first_neuron + ann->total_neurons + 1 + i;
 711                  * printf("candidate[%d] = activation: %s, steepness: %f, score: %f\n",
 712                  * i, FANN_ACTIVATIONFUNC_NAMES[cand->activation_function],
 713                  * cand->activation_steepness, ann->cascade_candidate_scores[i]); */
 714
 715                 if(ann->cascade_candidate_scores[i] > best_score)
 716                 {
 717                         best_candidate = i;
 718                         best_score = ann->cascade_candidate_scores[best_candidate];
 719                 }
 720         }
 721
 722         ann->cascade_best_candidate = ann->total_neurons + best_candidate + 1;
 723 #ifdef CASCADE_DEBUG_FULL
 724         printf("Best candidate[%d]: with score %f, real score: %f\n", best_candidate,
 725                    ann->MSE_value - best_score, best_score);
 726 #endif
 727
 728         return best_score;
 729 }
 730
 731 /* add a layer ad the position pointed to by *layer */
 732 struct fann_layer *fann_add_layer(struct fann *ann, struct fann_layer *layer)
 733 {
 734         int layer_pos = layer - ann->first_layer;
 735         int num_layers = ann->last_layer - ann->first_layer + 1;
 736         int i;
 737
 738         /* allocate the layer */
 739         struct fann_layer *layers =
 740                 (struct fann_layer *) realloc(ann->first_layer, num_layers * sizeof(struct fann_layer));
 741         if(layers == NULL)
 742         {
 743                 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
 744                 return NULL;
 745         }
 746
 747         /* copy layers so that the free space is at the right location */
 748         for(i = num_layers - 1; i >= layer_pos; i--)
 749         {
 750                 layers[i] = layers[i - 1];
 751         }
 752
 753         /* the newly allocated layer is empty */
 754         layers[layer_pos].first_neuron = layers[layer_pos + 1].first_neuron;
 755         layers[layer_pos].last_neuron = layers[layer_pos + 1].first_neuron;
 756
 757         /* Set the ann pointers correctly */
 758         ann->first_layer = layers;
 759         ann->last_layer = layers + num_layers;
 760
 761 #ifdef CASCADE_DEBUG_FULL
 762         printf("add layer at pos %d\n", layer_pos);
 763 #endif
 764
 765         return layers + layer_pos;
 766 }
 767
 768 void fann_set_shortcut_connections(struct fann *ann)
 769 {
 770         struct fann_layer *layer_it;
 771         struct fann_neuron *neuron_it, **neuron_pointers, *neurons;
 772         unsigned int num_connections = 0, i;
 773
 774         neuron_pointers = ann->connections;
 775         neurons = ann->first_layer->first_neuron;
 776
 777         for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
 778         {
 779                 for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++)
 780                 {
 781
 782                         neuron_pointers += num_connections;
 783                         num_connections = neuron_it->last_con - neuron_it->first_con;
 784
 785                         for(i = 0; i != num_connections; i++)
 786                         {
 787                                 neuron_pointers[i] = neurons + i;
 788                         }
 789                 }
 790         }
 791 }
 792
 793 void fann_add_candidate_neuron(struct fann *ann, struct fann_layer *layer)
 794 {
 795         unsigned int num_connections_in = layer->first_neuron - ann->first_layer->first_neuron;
 796         unsigned int num_connections_out =
 797                 (ann->last_layer - 1)->last_neuron - (layer + 1)->first_neuron;
 798         unsigned int num_connections_move = num_connections_out + num_connections_in;
 799
 800         unsigned int candidate_con, candidate_output_weight;
 801         int i;
 802
 803         struct fann_layer *layer_it;
 804         struct fann_neuron *neuron_it, *neuron_place, *candidate;
 805
 806         /* We know that there is enough room for the new neuron
 807          * (the candidates are in the same arrays), so move
 808          * the last neurons to make room for this neuron.
 809          */
 810
 811         /* first move the pointers to neurons in the layer structs */
 812         for(layer_it = ann->last_layer - 1; layer_it != layer; layer_it--)
 813         {
 814 #ifdef CASCADE_DEBUG_FULL
 815                 printf("move neuron pointers in layer %d, first(%d -> %d), last(%d -> %d)\n",
 816                            layer_it - ann->first_layer,
 817                            layer_it->first_neuron - ann->first_layer->first_neuron,
 818                            layer_it->first_neuron - ann->first_layer->first_neuron + 1,
 819                            layer_it->last_neuron - ann->first_layer->first_neuron,
 820                            layer_it->last_neuron - ann->first_layer->first_neuron + 1);
 821 #endif
 822                 layer_it->first_neuron++;
 823                 layer_it->last_neuron++;
 824         }
 825
 826         /* also move the last neuron in the layer that needs the neuron added */
 827         layer->last_neuron++;
 828
 829         /* this is the place that should hold the new neuron */
 830         neuron_place = layer->last_neuron - 1;
 831
 832 #ifdef CASCADE_DEBUG_FULL
 833         printf("num_connections_in=%d, num_connections_out=%d\n", num_connections_in,
 834                    num_connections_out);
 835 #endif
 836
 837         candidate = ann->first_layer->first_neuron + ann->cascade_best_candidate;
 838
 839         /* the output weights for the candidates are located after the input weights */
 840         candidate_output_weight = candidate->last_con;
 841
 842         /* move the actual output neurons and the indexes to the connection arrays */
 843         for(neuron_it = (ann->last_layer - 1)->last_neuron - 1; neuron_it != neuron_place; neuron_it--)
 844         {
 845 #ifdef CASCADE_DEBUG_FULL
 846                 printf("move neuron %d -> %d\n", neuron_it - ann->first_layer->first_neuron - 1,
 847                            neuron_it - ann->first_layer->first_neuron);
 848 #endif
 849                 *neuron_it = *(neuron_it - 1);
 850
 851                 /* move the weights */
 852 #ifdef CASCADE_DEBUG_FULL
 853                 printf("move weight[%d ... %d] -> weight[%d ... %d]\n", neuron_it->first_con,
 854                            neuron_it->last_con - 1, neuron_it->first_con + num_connections_move - 1,
 855                            neuron_it->last_con + num_connections_move - 2);
 856 #endif
 857                 for(i = neuron_it->last_con - 1; i >= (int)neuron_it->first_con; i--)
 858                 {
 859 #ifdef CASCADE_DEBUG_FULL
 860                         printf("move weight[%d] = weight[%d]\n", i + num_connections_move - 1, i);
 861 #endif
 862                         ann->weights[i + num_connections_move - 1] = ann->weights[i];
 863                 }
 864
 865                 /* move the indexes to weights */
 866                 neuron_it->last_con += num_connections_move;
 867                 num_connections_move--;
 868                 neuron_it->first_con += num_connections_move;
 869
 870                 /* set the new weight to the newly allocated neuron */
 871                 ann->weights[neuron_it->last_con - 1] =
 872                         (ann->weights[candidate_output_weight]) * ann->cascade_weight_multiplier;
 873                 candidate_output_weight++;
 874         }
 875
 876         /* Now inititalize the actual neuron */
 877         neuron_place->value = 0;
 878         neuron_place->sum = 0;
 879         neuron_place->activation_function = candidate->activation_function;
 880         neuron_place->activation_steepness = candidate->activation_steepness;
 881         neuron_place->last_con = (neuron_place + 1)->first_con;
 882         neuron_place->first_con = neuron_place->last_con - num_connections_in;
 883 #ifdef CASCADE_DEBUG_FULL
 884         printf("neuron[%d] = weights[%d ... %d] activation: %s, steepness: %f\n",
 885                    neuron_place - ann->first_layer->first_neuron, neuron_place->first_con,
 886                    neuron_place->last_con - 1, FANN_ACTIVATIONFUNC_NAMES[neuron_place->activation_function],
 887                    neuron_place->activation_steepness);/* TODO remove */
 888 #endif
 889
 890         candidate_con = candidate->first_con;
 891         /* initialize the input weights at random */
 892 #ifdef CASCADE_DEBUG_FULL
 893         printf("move cand weights[%d ... %d] -> [%d ... %d]\n", candidate_con,
 894                    candidate_con + num_connections_in - 1, neuron_place->first_con,
 895                    neuron_place->last_con - 1);
 896 #endif
 897
 898         for(i = 0; i < (int)num_connections_in; i++)
 899         {
 900                 ann->weights[i + neuron_place->first_con] = ann->weights[i + candidate_con];
 901 #ifdef CASCADE_DEBUG_FULL
 902                 printf("move weights[%d] -> weights[%d] (%f)\n", i + candidate_con,
 903                            i + neuron_place->first_con, ann->weights[i + neuron_place->first_con]);
 904 #endif
 905         }
 906
 907         /* Change some of main variables */
 908         ann->total_neurons++;
 909         ann->total_connections += num_connections_in + num_connections_out;
 910
 911         return;
 912 }
 913
 914 void fann_install_candidate(struct fann *ann)
 915 {
 916         struct fann_layer *layer;
 917
 918         layer = fann_add_layer(ann, ann->last_layer - 1);
 919         fann_add_candidate_neuron(ann, layer);
 920         return;
 921 }
 922
 923 #endif /* FIXEDFANN */
 924
 925 FANN_EXTERNAL unsigned int FANN_API fann_get_cascade_num_candidates(struct fann *ann)
 926 {
 927         return ann->cascade_activation_functions_count *
 928                 ann->cascade_activation_steepnesses_count *
 929                 ann->cascade_num_candidate_groups;
 930 }
 931
 932 FANN_GET_SET(float, cascade_output_change_fraction)
 933 FANN_GET_SET(unsigned int, cascade_output_stagnation_epochs)
 934 FANN_GET_SET(float, cascade_candidate_change_fraction)
 935 FANN_GET_SET(unsigned int, cascade_candidate_stagnation_epochs)
 936 FANN_GET_SET(unsigned int, cascade_num_candidate_groups)
 937 FANN_GET_SET(fann_type, cascade_weight_multiplier)
 938 FANN_GET_SET(fann_type, cascade_candidate_limit)
 939 FANN_GET_SET(unsigned int, cascade_max_out_epochs)
 940 FANN_GET_SET(unsigned int, cascade_max_cand_epochs)
 941
 942 FANN_GET(unsigned int, cascade_activation_functions_count)
 943 FANN_GET(enum fann_activationfunc_enum *, cascade_activation_functions)
 944
 945 FANN_EXTERNAL void fann_set_cascade_activation_functions(struct fann *ann,
 946                                                                                                                  enum fann_activationfunc_enum *
 947                                                                                                                  cascade_activation_functions,
 948                                                                                                                  unsigned int
 949                                                                                                                  cascade_activation_functions_count)
 950 {
 951         if(ann->cascade_activation_functions_count != cascade_activation_functions_count)
 952         {
 953                 ann->cascade_activation_functions_count = cascade_activation_functions_count;
 954
 955                 /* reallocate mem */
 956                 ann->cascade_activation_functions =
 957                         (enum fann_activationfunc_enum *)realloc(ann->cascade_activation_functions,
 958                         ann->cascade_activation_functions_count * sizeof(enum fann_activationfunc_enum));
 959                 if(ann->cascade_activation_functions == NULL)
 960                 {
 961                         fann_error((struct fann_error*)ann, FANN_E_CANT_ALLOCATE_MEM);
 962                         return;
 963                 }
 964         }
 965
 966         memmove(ann->cascade_activation_functions, cascade_activation_functions,
 967                 ann->cascade_activation_functions_count * sizeof(enum fann_activationfunc_enum));
 968 }
 969
 970 FANN_GET(unsigned int, cascade_activation_steepnesses_count)
 971 FANN_GET(fann_type *, cascade_activation_steepnesses)
 972
 973 FANN_EXTERNAL void fann_set_cascade_activation_steepnesses(struct fann *ann,
 974                                                                                                                    fann_type *
 975                                                                                                                    cascade_activation_steepnesses,
 976                                                                                                                    unsigned int
 977                                                                                                                    cascade_activation_steepnesses_count)
 978 {
 979         if(ann->cascade_activation_steepnesses_count != cascade_activation_steepnesses_count)
 980         {
 981                 ann->cascade_activation_steepnesses_count = cascade_activation_steepnesses_count;
 982
 983                 /* reallocate mem */
 984                 ann->cascade_activation_steepnesses =
 985                         (fann_type *)realloc(ann->cascade_activation_steepnesses,
 986                         ann->cascade_activation_steepnesses_count * sizeof(fann_type));
 987                 if(ann->cascade_activation_steepnesses == NULL)
 988                 {
 989                         fann_error((struct fann_error*)ann, FANN_E_CANT_ALLOCATE_MEM);
 990                         return;
 991                 }
 992         }
 993
 994         memmove(ann->cascade_activation_steepnesses, cascade_activation_steepnesses,
 995                 ann->cascade_activation_steepnesses_count * sizeof(fann_type));
 996 }