2 Fast Artificial Neural Network Library (fann)
3 Copyright (C) 2003 Steffen Nissen (lukesky@diku.dk)
5 This library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 This library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with this library; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 /* #define CASCADE_DEBUG */
27 /* #define CASCADE_DEBUG_FULL */
29 void fann_print_connections_raw(struct fann *ann)
33 for(i = 0; i < ann->total_connections_allocated; i++)
35 if(i == ann->total_connections)
39 printf("%f ", ann->weights[i]);
44 /* Cascade training directly on the training data.
45 The connected_neurons pointers are not valid during training,
46 but they will be again after training.
48 FANN_EXTERNAL void FANN_API fann_cascadetrain_on_data(struct fann *ann, struct fann_train_data *data,
49 unsigned int max_neurons,
50 unsigned int neurons_between_reports,
55 unsigned int total_epochs = 0;
56 int desired_error_reached;
58 if(neurons_between_reports && ann->callback == NULL)
60 printf("Max neurons %3d. Desired error: %.6f\n", max_neurons, desired_error);
63 for(i = 1; i <= max_neurons; i++)
65 /* train output neurons */
66 total_epochs += fann_train_outputs(ann, data, desired_error);
67 error = fann_get_MSE(ann);
68 desired_error_reached = fann_desired_error_reached(ann, desired_error);
70 /* print current error */
71 if(neurons_between_reports &&
72 (i % neurons_between_reports == 0
73 || i == max_neurons || i == 1 || desired_error_reached == 0))
75 if(ann->callback == NULL)
78 ("Neurons %3d. Current error: %.6f. Total error:%8.4f. Epochs %5d. Bit fail %3d",
79 i, error, ann->MSE_value, total_epochs, ann->num_bit_fail);
80 if((ann->last_layer-2) != ann->first_layer)
82 printf(". candidate steepness %.2f. function %s",
83 (ann->last_layer-2)->first_neuron->activation_steepness,
84 FANN_ACTIVATIONFUNC_NAMES[(ann->last_layer-2)->first_neuron->activation_function]);
88 else if((*ann->callback) (ann, data, max_neurons,
89 neurons_between_reports, desired_error, total_epochs) == -1)
91 /* you can break the training by returning -1 */
96 if(desired_error_reached == 0)
99 if(fann_initialize_candidates(ann) == -1)
101 /* Unable to initialize room for candidates */
105 /* train new candidates */
106 total_epochs += fann_train_candidates(ann, data);
108 /* this installs the best candidate */
109 fann_install_candidate(ann);
112 /* Train outputs one last time but without any desired error */
113 total_epochs += fann_train_outputs(ann, data, 0.0);
115 if(neurons_between_reports && ann->callback == NULL)
117 printf("Train outputs Current error: %.6f. Epochs %6d\n", fann_get_MSE(ann),
121 /* Set pointers in connected_neurons
122 * This is ONLY done in the end of cascade training,
123 * since there is no need for them during training.
125 fann_set_shortcut_connections(ann);
128 FANN_EXTERNAL void FANN_API fann_cascadetrain_on_file(struct fann *ann, const char *filename,
129 unsigned int max_neurons,
130 unsigned int neurons_between_reports,
133 struct fann_train_data *data = fann_read_train_from_file(filename);
139 fann_cascadetrain_on_data(ann, data, max_neurons, neurons_between_reports, desired_error);
140 fann_destroy_train(data);
143 int fann_train_outputs(struct fann *ann, struct fann_train_data *data, float desired_error)
145 float error, initial_error, error_improvement;
146 float target_improvement = 0.0;
147 float backslide_improvement = -1.0e20f;
149 unsigned int max_epochs = ann->cascade_max_out_epochs;
150 unsigned int stagnation = max_epochs;
152 /* TODO should perhaps not clear all arrays */
153 fann_clear_train_arrays(ann);
155 /* run an initial epoch to set the initital error */
156 initial_error = fann_train_outputs_epoch(ann, data);
158 if(fann_desired_error_reached(ann, desired_error) == 0)
161 for(i = 1; i < max_epochs; i++)
163 error = fann_train_outputs_epoch(ann, data);
165 /*printf("Epoch %6d. Current error: %.6f. Bit fail %d.\n", i, error, ann->num_bit_fail); */
167 if(fann_desired_error_reached(ann, desired_error) == 0)
170 printf("Error %f < %f\n", error, desired_error);
175 /* Improvement since start of train */
176 error_improvement = initial_error - error;
178 /* After any significant change, set a new goal and
179 * allow a new quota of epochs to reach it */
180 if((error_improvement > target_improvement) || (error_improvement < backslide_improvement))
182 /*printf("error_improvement=%f, target_improvement=%f, backslide_improvement=%f, stagnation=%d\n", error_improvement, target_improvement, backslide_improvement, stagnation); */
184 target_improvement = error_improvement * (1.0f + ann->cascade_output_change_fraction);
185 backslide_improvement = error_improvement * (1.0f - ann->cascade_output_change_fraction);
186 stagnation = i + ann->cascade_output_stagnation_epochs;
189 /* No improvement in allotted period, so quit */
199 float fann_train_outputs_epoch(struct fann *ann, struct fann_train_data *data)
205 for(i = 0; i < data->num_data; i++)
207 fann_run(ann, data->input[i]);
208 fann_compute_MSE(ann, data->output[i]);
209 fann_update_slopes_batch(ann, ann->last_layer - 1, ann->last_layer - 1);
212 switch (ann->training_algorithm)
214 case FANN_TRAIN_RPROP:
215 fann_update_weights_irpropm(ann, (ann->last_layer - 1)->first_neuron->first_con,
216 ann->total_connections);
218 case FANN_TRAIN_QUICKPROP:
219 fann_update_weights_quickprop(ann, data->num_data,
220 (ann->last_layer - 1)->first_neuron->first_con,
221 ann->total_connections);
223 case FANN_TRAIN_BATCH:
224 case FANN_TRAIN_INCREMENTAL:
225 fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
228 return fann_get_MSE(ann);
231 int fann_reallocate_connections(struct fann *ann, unsigned int total_connections)
233 /* The connections are allocated, but the pointers inside are
234 * first moved in the end of the cascade training session.
238 printf("realloc from %d to %d\n", ann->total_connections_allocated, total_connections);
241 (struct fann_neuron **) realloc(ann->connections,
242 total_connections * sizeof(struct fann_neuron *));
243 if(ann->connections == NULL)
245 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
249 ann->weights = (fann_type *) realloc(ann->weights, total_connections * sizeof(fann_type));
250 if(ann->weights == NULL)
252 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
257 (fann_type *) realloc(ann->train_slopes, total_connections * sizeof(fann_type));
258 if(ann->train_slopes == NULL)
260 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
264 ann->prev_steps = (fann_type *) realloc(ann->prev_steps, total_connections * sizeof(fann_type));
265 if(ann->prev_steps == NULL)
267 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
271 ann->prev_train_slopes =
272 (fann_type *) realloc(ann->prev_train_slopes, total_connections * sizeof(fann_type));
273 if(ann->prev_train_slopes == NULL)
275 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
279 ann->total_connections_allocated = total_connections;
284 int fann_reallocate_neurons(struct fann *ann, unsigned int total_neurons)
286 struct fann_layer *layer_it;
287 struct fann_neuron *neurons;
288 unsigned int num_neurons = 0;
289 unsigned int num_neurons_so_far = 0;
292 (struct fann_neuron *) realloc(ann->first_layer->first_neuron,
293 total_neurons * sizeof(struct fann_neuron));
294 ann->total_neurons_allocated = total_neurons;
298 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
302 /* Also allocate room for more train_errors */
303 ann->train_errors = (fann_type *) realloc(ann->train_errors, total_neurons * sizeof(fann_type));
304 if(ann->train_errors == NULL)
306 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
310 if(neurons != ann->first_layer->first_neuron)
312 /* Then the memory has moved, also move the pointers */
314 #ifdef CASCADE_DEBUG_FULL
315 printf("Moving neuron pointers\n");
318 /* Move pointers from layers to neurons */
319 for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++)
321 num_neurons = layer_it->last_neuron - layer_it->first_neuron;
322 layer_it->first_neuron = neurons + num_neurons_so_far;
323 layer_it->last_neuron = layer_it->first_neuron + num_neurons;
324 num_neurons_so_far += num_neurons;
331 int fann_initialize_candidates(struct fann *ann)
333 /* The candidates are allocated after the normal neurons and connections,
334 * but there is an empty place between the real neurons and the candidate neurons,
335 * so that it will be possible to make room when the chosen candidate are copied in
336 * on the desired place.
338 unsigned int neurons_to_allocate, connections_to_allocate;
339 unsigned int num_candidates = fann_get_cascade_num_candidates(ann);
340 unsigned int num_neurons = ann->total_neurons + num_candidates + 1;
341 unsigned int candidate_connections_in = ann->total_neurons - ann->num_output;
342 unsigned int candidate_connections_out = ann->num_output;
344 /* the number of connections going into a and out of a candidate is
345 * ann->total_neurons */
346 unsigned int num_connections =
347 ann->total_connections + (ann->total_neurons * (num_candidates + 1));
348 unsigned int first_candidate_connection = ann->total_connections + ann->total_neurons;
349 unsigned int first_candidate_neuron = ann->total_neurons + 1;
350 unsigned int connection_it, i, j, k, candidate_index;
351 struct fann_neuron *neurons;
352 fann_type initial_slope;
354 /* First make sure that there is enough room, and if not then allocate a
355 * bit more so that we do not need to allocate more room each time.
357 if(num_neurons > ann->total_neurons_allocated)
359 /* Then we need to allocate more neurons
360 * Allocate half as many neurons as already exist (at least ten)
362 neurons_to_allocate = num_neurons + num_neurons / 2;
363 if(neurons_to_allocate < num_neurons + 10)
365 neurons_to_allocate = num_neurons + 10;
368 if(fann_reallocate_neurons(ann, neurons_to_allocate) == -1)
374 if(num_connections > ann->total_connections_allocated)
376 /* Then we need to allocate more connections
377 * Allocate half as many connections as already exist
378 * (at least enough for ten neurons)
380 connections_to_allocate = num_connections + num_connections / 2;
381 if(connections_to_allocate < num_connections + ann->total_neurons * 10)
383 connections_to_allocate = num_connections + ann->total_neurons * 10;
386 if(fann_reallocate_connections(ann, connections_to_allocate) == -1)
394 connection_it = first_candidate_connection;
395 neurons = ann->first_layer->first_neuron;
396 candidate_index = first_candidate_neuron;
398 for(i = 0; i < ann->cascade_activation_functions_count; i++)
400 for(j = 0; j < ann->cascade_activation_steepnesses_count; j++)
402 for(k = 0; k < ann->cascade_num_candidate_groups; k++)
404 /* TODO candidates should actually be created both in
405 * the last layer before the output layer, and in a new layer.
407 neurons[candidate_index].value = 0;
408 neurons[candidate_index].sum = 0;
410 neurons[candidate_index].activation_function =
411 ann->cascade_activation_functions[i];
412 neurons[candidate_index].activation_steepness =
413 ann->cascade_activation_steepnesses[j];
415 neurons[candidate_index].first_con = connection_it;
416 connection_it += candidate_connections_in;
417 neurons[candidate_index].last_con = connection_it;
418 /* We have no specific pointers to the output weights, but they are
419 * available after last_con */
420 connection_it += candidate_connections_out;
421 ann->train_errors[candidate_index] = 0;
427 /* Now randomize the weights and zero out the arrays that needs zeroing out.
429 #ifdef CASCADE_DEBUG_FULL
430 printf("random cand weight [%d ... %d]\n", first_candidate_connection, num_connections - 1);
432 if(ann->training_algorithm == FANN_TRAIN_RPROP)
434 initial_slope = ann->rprop_delta_zero;
440 for(i = first_candidate_connection; i < num_connections; i++)
442 ann->weights[i] = fann_random_weight();
443 /*ann->weights[i] = fann_rand(-0.25,0.25);*/
444 ann->train_slopes[i] = 0;
445 ann->prev_steps[i] = 0;
446 ann->prev_train_slopes[i] = initial_slope;
452 int fann_train_candidates(struct fann *ann, struct fann_train_data *data)
454 fann_type best_cand_score = 0.0;
455 fann_type target_cand_score = 0.0;
456 fann_type backslide_cand_score = -1.0e20f;
458 unsigned int max_epochs = ann->cascade_max_cand_epochs;
459 unsigned int stagnation = max_epochs;
461 if(ann->cascade_candidate_scores == NULL)
463 ann->cascade_candidate_scores =
464 (fann_type *) malloc(fann_get_cascade_num_candidates(ann) * sizeof(fann_type));
465 if(ann->cascade_candidate_scores == NULL)
467 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
472 for(i = 0; i < max_epochs; i++)
474 best_cand_score = fann_train_candidates_epoch(ann, data);
476 if(best_cand_score / ann->MSE_value > ann->cascade_candidate_limit)
479 printf("above candidate limit %f/%f > %f", best_cand_score, ann->MSE_value,
480 ann->cascade_candidate_limit);
485 if((best_cand_score > target_cand_score) || (best_cand_score < backslide_cand_score))
487 #ifdef CASCADE_DEBUG_FULL
488 printf("Best candidate score %f, real score: %f\n", ann->MSE_value - best_cand_score,
490 /* printf("best_cand_score=%f, target_cand_score=%f, backslide_cand_score=%f, stagnation=%d\n", best_cand_score, target_cand_score, backslide_cand_score, stagnation); */
493 target_cand_score = best_cand_score * (1.0f + ann->cascade_candidate_change_fraction);
494 backslide_cand_score = best_cand_score * (1.0f - ann->cascade_candidate_change_fraction);
495 stagnation = i + ann->cascade_candidate_stagnation_epochs;
498 /* No improvement in allotted period, so quit */
502 printf("Stagnation with %d epochs, best candidate score %f, real score: %f\n", i + 1,
503 ann->MSE_value - best_cand_score, best_cand_score);
510 printf("Max epochs %d reached, best candidate score %f, real score: %f\n", max_epochs,
511 ann->MSE_value - best_cand_score, best_cand_score);
516 void fann_update_candidate_slopes(struct fann *ann)
518 struct fann_neuron *neurons = ann->first_layer->first_neuron;
519 struct fann_neuron *first_cand = neurons + ann->total_neurons + 1;
520 struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann);
521 struct fann_neuron *cand_it;
522 unsigned int i, j, num_connections;
523 unsigned int num_output = ann->num_output;
524 fann_type max_sum, cand_sum, activation, derived, error_value, diff, cand_score;
525 fann_type *weights, *cand_out_weights, *cand_slopes, *cand_out_slopes;
526 fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
528 for(cand_it = first_cand; cand_it < last_cand; cand_it++)
530 cand_score = ann->cascade_candidate_scores[cand_it - first_cand];
533 /* code more or less stolen from fann_run to fast forward pass
536 num_connections = cand_it->last_con - cand_it->first_con;
537 weights = ann->weights + cand_it->first_con;
539 /* unrolled loop start */
540 i = num_connections & 3; /* same as modulo 4 */
544 cand_sum += weights[2] * neurons[2].value;
546 cand_sum += weights[1] * neurons[1].value;
548 cand_sum += weights[0] * neurons[0].value;
553 for(; i != num_connections; i += 4)
556 weights[i] * neurons[i].value +
557 weights[i + 1] * neurons[i + 1].value +
558 weights[i + 2] * neurons[i + 2].value + weights[i + 3] * neurons[i + 3].value;
561 * for(i = 0; i < num_connections; i++){
562 * cand_sum += weights[i] * neurons[i].value;
565 /* unrolled loop end */
567 max_sum = 150/cand_it->activation_steepness;
568 if(cand_sum > max_sum)
570 else if(cand_sum < -max_sum)
574 fann_activation(ann, cand_it->activation_function, cand_it->activation_steepness,
576 /* printf("%f = sigmoid(%f);\n", activation, cand_sum); */
578 cand_it->sum = cand_sum;
579 cand_it->value = activation;
581 derived = fann_activation_derived(cand_it->activation_function,
582 cand_it->activation_steepness, activation, cand_sum);
584 /* The output weights is located right after the input weights in
587 cand_out_weights = weights + num_connections;
589 cand_out_slopes = ann->train_slopes + cand_it->first_con + num_connections;
590 for(j = 0; j < num_output; j++)
592 diff = (activation * cand_out_weights[j]) - output_train_errors[j];
593 #ifdef CASCADE_DEBUG_FULL
594 /* printf("diff = %f = (%f * %f) - %f;\n", diff, activation, cand_out_weights[j], output_train_errors[j]); */
596 cand_out_slopes[j] -= 2.0f * diff * activation;
597 #ifdef CASCADE_DEBUG_FULL
598 /* printf("cand_out_slopes[%d] <= %f += %f * %f;\n", j, cand_out_slopes[j], diff, activation); */
600 error_value += diff * cand_out_weights[j];
601 cand_score -= (diff * diff);
602 #ifdef CASCADE_DEBUG_FULL
603 /* printf("cand_score[%d][%d] = %f -= (%f * %f)\n", cand_it - first_cand, j, cand_score, diff, diff); */
605 printf("cand[%d]: error=%f, activation=%f, diff=%f, slope=%f\n", cand_it - first_cand,
606 output_train_errors[j], (activation * cand_out_weights[j]), diff,
607 -2.0 * diff * activation);
611 ann->cascade_candidate_scores[cand_it - first_cand] = cand_score;
612 error_value *= derived;
614 cand_slopes = ann->train_slopes + cand_it->first_con;
615 for(i = 0; i < num_connections; i++)
617 cand_slopes[i] -= error_value * neurons[i].value;
622 void fann_update_candidate_weights(struct fann *ann, unsigned int num_data)
624 struct fann_neuron *first_cand = (ann->last_layer - 1)->last_neuron + 1; /* there is an empty neuron between the actual neurons and the candidate neuron */
625 struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann) - 1;
627 switch (ann->training_algorithm)
629 case FANN_TRAIN_RPROP:
630 fann_update_weights_irpropm(ann, first_cand->first_con,
631 last_cand->last_con + ann->num_output);
633 case FANN_TRAIN_QUICKPROP:
634 fann_update_weights_quickprop(ann, num_data, first_cand->first_con,
635 last_cand->last_con + ann->num_output);
637 case FANN_TRAIN_BATCH:
638 case FANN_TRAIN_INCREMENTAL:
639 fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
644 fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data *data)
647 unsigned int best_candidate;
648 fann_type best_score;
649 unsigned int num_cand = fann_get_cascade_num_candidates(ann);
650 fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
651 struct fann_neuron *output_neurons = (ann->last_layer - 1)->first_neuron;
653 for(i = 0; i < num_cand; i++)
655 /* The ann->MSE_value is actually the sum squared error */
656 ann->cascade_candidate_scores[i] = ann->MSE_value;
658 /*printf("start score: %f\n", ann->MSE_value); */
660 for(i = 0; i < data->num_data; i++)
662 fann_run(ann, data->input[i]);
664 for(j = 0; j < ann->num_output; j++)
666 /* TODO only debug, but the error is in opposite direction, this might be usefull info */
667 /* if(output_train_errors[j] != (ann->output[j] - data->output[i][j])){
668 * printf("difference in calculated error at %f != %f; %f = %f - %f;\n", output_train_errors[j], (ann->output[j] - data->output[i][j]), output_train_errors[j], ann->output[j], data->output[i][j]);
672 * output_train_errors[j] = (data->output[i][j] - ann->output[j])/2;
673 * output_train_errors[j] = ann->output[j] - data->output[i][j];
676 output_train_errors[j] = (data->output[i][j] - ann->output[j]);
678 switch (output_neurons[j].activation_function)
680 case FANN_LINEAR_PIECE_SYMMETRIC:
681 case FANN_SIGMOID_SYMMETRIC:
682 case FANN_SIGMOID_SYMMETRIC_STEPWISE:
683 case FANN_THRESHOLD_SYMMETRIC:
684 case FANN_ELLIOT_SYMMETRIC:
685 case FANN_GAUSSIAN_SYMMETRIC:
686 output_train_errors[j] /= 2.0;
691 case FANN_SIGMOID_STEPWISE:
693 case FANN_GAUSSIAN_STEPWISE:
695 case FANN_LINEAR_PIECE:
700 fann_update_candidate_slopes(ann);
703 fann_update_candidate_weights(ann, data->num_data);
705 /* find the best candidate score */
707 best_score = ann->cascade_candidate_scores[best_candidate];
708 for(i = 1; i < num_cand; i++)
710 /*struct fann_neuron *cand = ann->first_layer->first_neuron + ann->total_neurons + 1 + i;
711 * printf("candidate[%d] = activation: %s, steepness: %f, score: %f\n",
712 * i, FANN_ACTIVATIONFUNC_NAMES[cand->activation_function],
713 * cand->activation_steepness, ann->cascade_candidate_scores[i]); */
715 if(ann->cascade_candidate_scores[i] > best_score)
718 best_score = ann->cascade_candidate_scores[best_candidate];
722 ann->cascade_best_candidate = ann->total_neurons + best_candidate + 1;
723 #ifdef CASCADE_DEBUG_FULL
724 printf("Best candidate[%d]: with score %f, real score: %f\n", best_candidate,
725 ann->MSE_value - best_score, best_score);
731 /* add a layer ad the position pointed to by *layer */
732 struct fann_layer *fann_add_layer(struct fann *ann, struct fann_layer *layer)
734 int layer_pos = layer - ann->first_layer;
735 int num_layers = ann->last_layer - ann->first_layer + 1;
738 /* allocate the layer */
739 struct fann_layer *layers =
740 (struct fann_layer *) realloc(ann->first_layer, num_layers * sizeof(struct fann_layer));
743 fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
747 /* copy layers so that the free space is at the right location */
748 for(i = num_layers - 1; i >= layer_pos; i--)
750 layers[i] = layers[i - 1];
753 /* the newly allocated layer is empty */
754 layers[layer_pos].first_neuron = layers[layer_pos + 1].first_neuron;
755 layers[layer_pos].last_neuron = layers[layer_pos + 1].first_neuron;
757 /* Set the ann pointers correctly */
758 ann->first_layer = layers;
759 ann->last_layer = layers + num_layers;
761 #ifdef CASCADE_DEBUG_FULL
762 printf("add layer at pos %d\n", layer_pos);
765 return layers + layer_pos;
768 void fann_set_shortcut_connections(struct fann *ann)
770 struct fann_layer *layer_it;
771 struct fann_neuron *neuron_it, **neuron_pointers, *neurons;
772 unsigned int num_connections = 0, i;
774 neuron_pointers = ann->connections;
775 neurons = ann->first_layer->first_neuron;
777 for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
779 for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++)
782 neuron_pointers += num_connections;
783 num_connections = neuron_it->last_con - neuron_it->first_con;
785 for(i = 0; i != num_connections; i++)
787 neuron_pointers[i] = neurons + i;
793 void fann_add_candidate_neuron(struct fann *ann, struct fann_layer *layer)
795 unsigned int num_connections_in = layer->first_neuron - ann->first_layer->first_neuron;
796 unsigned int num_connections_out =
797 (ann->last_layer - 1)->last_neuron - (layer + 1)->first_neuron;
798 unsigned int num_connections_move = num_connections_out + num_connections_in;
800 unsigned int candidate_con, candidate_output_weight;
803 struct fann_layer *layer_it;
804 struct fann_neuron *neuron_it, *neuron_place, *candidate;
806 /* We know that there is enough room for the new neuron
807 * (the candidates are in the same arrays), so move
808 * the last neurons to make room for this neuron.
811 /* first move the pointers to neurons in the layer structs */
812 for(layer_it = ann->last_layer - 1; layer_it != layer; layer_it--)
814 #ifdef CASCADE_DEBUG_FULL
815 printf("move neuron pointers in layer %d, first(%d -> %d), last(%d -> %d)\n",
816 layer_it - ann->first_layer,
817 layer_it->first_neuron - ann->first_layer->first_neuron,
818 layer_it->first_neuron - ann->first_layer->first_neuron + 1,
819 layer_it->last_neuron - ann->first_layer->first_neuron,
820 layer_it->last_neuron - ann->first_layer->first_neuron + 1);
822 layer_it->first_neuron++;
823 layer_it->last_neuron++;
826 /* also move the last neuron in the layer that needs the neuron added */
827 layer->last_neuron++;
829 /* this is the place that should hold the new neuron */
830 neuron_place = layer->last_neuron - 1;
832 #ifdef CASCADE_DEBUG_FULL
833 printf("num_connections_in=%d, num_connections_out=%d\n", num_connections_in,
834 num_connections_out);
837 candidate = ann->first_layer->first_neuron + ann->cascade_best_candidate;
839 /* the output weights for the candidates are located after the input weights */
840 candidate_output_weight = candidate->last_con;
842 /* move the actual output neurons and the indexes to the connection arrays */
843 for(neuron_it = (ann->last_layer - 1)->last_neuron - 1; neuron_it != neuron_place; neuron_it--)
845 #ifdef CASCADE_DEBUG_FULL
846 printf("move neuron %d -> %d\n", neuron_it - ann->first_layer->first_neuron - 1,
847 neuron_it - ann->first_layer->first_neuron);
849 *neuron_it = *(neuron_it - 1);
851 /* move the weights */
852 #ifdef CASCADE_DEBUG_FULL
853 printf("move weight[%d ... %d] -> weight[%d ... %d]\n", neuron_it->first_con,
854 neuron_it->last_con - 1, neuron_it->first_con + num_connections_move - 1,
855 neuron_it->last_con + num_connections_move - 2);
857 for(i = neuron_it->last_con - 1; i >= (int)neuron_it->first_con; i--)
859 #ifdef CASCADE_DEBUG_FULL
860 printf("move weight[%d] = weight[%d]\n", i + num_connections_move - 1, i);
862 ann->weights[i + num_connections_move - 1] = ann->weights[i];
865 /* move the indexes to weights */
866 neuron_it->last_con += num_connections_move;
867 num_connections_move--;
868 neuron_it->first_con += num_connections_move;
870 /* set the new weight to the newly allocated neuron */
871 ann->weights[neuron_it->last_con - 1] =
872 (ann->weights[candidate_output_weight]) * ann->cascade_weight_multiplier;
873 candidate_output_weight++;
876 /* Now inititalize the actual neuron */
877 neuron_place->value = 0;
878 neuron_place->sum = 0;
879 neuron_place->activation_function = candidate->activation_function;
880 neuron_place->activation_steepness = candidate->activation_steepness;
881 neuron_place->last_con = (neuron_place + 1)->first_con;
882 neuron_place->first_con = neuron_place->last_con - num_connections_in;
883 #ifdef CASCADE_DEBUG_FULL
884 printf("neuron[%d] = weights[%d ... %d] activation: %s, steepness: %f\n",
885 neuron_place - ann->first_layer->first_neuron, neuron_place->first_con,
886 neuron_place->last_con - 1, FANN_ACTIVATIONFUNC_NAMES[neuron_place->activation_function],
887 neuron_place->activation_steepness);/* TODO remove */
890 candidate_con = candidate->first_con;
891 /* initialize the input weights at random */
892 #ifdef CASCADE_DEBUG_FULL
893 printf("move cand weights[%d ... %d] -> [%d ... %d]\n", candidate_con,
894 candidate_con + num_connections_in - 1, neuron_place->first_con,
895 neuron_place->last_con - 1);
898 for(i = 0; i < (int)num_connections_in; i++)
900 ann->weights[i + neuron_place->first_con] = ann->weights[i + candidate_con];
901 #ifdef CASCADE_DEBUG_FULL
902 printf("move weights[%d] -> weights[%d] (%f)\n", i + candidate_con,
903 i + neuron_place->first_con, ann->weights[i + neuron_place->first_con]);
907 /* Change some of main variables */
908 ann->total_neurons++;
909 ann->total_connections += num_connections_in + num_connections_out;
914 void fann_install_candidate(struct fann *ann)
916 struct fann_layer *layer;
918 layer = fann_add_layer(ann, ann->last_layer - 1);
919 fann_add_candidate_neuron(ann, layer);
923 #endif /* FIXEDFANN */
925 FANN_EXTERNAL unsigned int FANN_API fann_get_cascade_num_candidates(struct fann *ann)
927 return ann->cascade_activation_functions_count *
928 ann->cascade_activation_steepnesses_count *
929 ann->cascade_num_candidate_groups;
932 FANN_GET_SET(float, cascade_output_change_fraction)
933 FANN_GET_SET(unsigned int, cascade_output_stagnation_epochs)
934 FANN_GET_SET(float, cascade_candidate_change_fraction)
935 FANN_GET_SET(unsigned int, cascade_candidate_stagnation_epochs)
936 FANN_GET_SET(unsigned int, cascade_num_candidate_groups)
937 FANN_GET_SET(fann_type, cascade_weight_multiplier)
938 FANN_GET_SET(fann_type, cascade_candidate_limit)
939 FANN_GET_SET(unsigned int, cascade_max_out_epochs)
940 FANN_GET_SET(unsigned int, cascade_max_cand_epochs)
942 FANN_GET(unsigned int, cascade_activation_functions_count)
943 FANN_GET(enum fann_activationfunc_enum *, cascade_activation_functions)
945 FANN_EXTERNAL void fann_set_cascade_activation_functions(struct fann *ann,
946 enum fann_activationfunc_enum *
947 cascade_activation_functions,
949 cascade_activation_functions_count)
951 if(ann->cascade_activation_functions_count != cascade_activation_functions_count)
953 ann->cascade_activation_functions_count = cascade_activation_functions_count;
956 ann->cascade_activation_functions =
957 (enum fann_activationfunc_enum *)realloc(ann->cascade_activation_functions,
958 ann->cascade_activation_functions_count * sizeof(enum fann_activationfunc_enum));
959 if(ann->cascade_activation_functions == NULL)
961 fann_error((struct fann_error*)ann, FANN_E_CANT_ALLOCATE_MEM);
966 memmove(ann->cascade_activation_functions, cascade_activation_functions,
967 ann->cascade_activation_functions_count * sizeof(enum fann_activationfunc_enum));
970 FANN_GET(unsigned int, cascade_activation_steepnesses_count)
971 FANN_GET(fann_type *, cascade_activation_steepnesses)
973 FANN_EXTERNAL void fann_set_cascade_activation_steepnesses(struct fann *ann,
975 cascade_activation_steepnesses,
977 cascade_activation_steepnesses_count)
979 if(ann->cascade_activation_steepnesses_count != cascade_activation_steepnesses_count)
981 ann->cascade_activation_steepnesses_count = cascade_activation_steepnesses_count;
984 ann->cascade_activation_steepnesses =
985 (fann_type *)realloc(ann->cascade_activation_steepnesses,
986 ann->cascade_activation_steepnesses_count * sizeof(fann_type));
987 if(ann->cascade_activation_steepnesses == NULL)
989 fann_error((struct fann_error*)ann, FANN_E_CANT_ALLOCATE_MEM);
994 memmove(ann->cascade_activation_steepnesses, cascade_activation_steepnesses,
995 ann->cascade_activation_steepnesses_count * sizeof(fann_type));