2 Fast Artificial Neural Network Library (fann)
3 Copyright (C) 2003 Steffen Nissen (lukesky@diku.dk)
5 This library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 This library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with this library; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #ifndef __fann_data_h__
21 #define __fann_data_h__
25 /* Section: FANN Datatypes
27 The two main datatypes used in the fann library is <struct fann>,
28 which represents an artificial neural network, and <struct fann_train_data>,
29 which represent training data.
34 fann_type is the type used for the weights, inputs and outputs of the neural network.
36 fann_type is defined as a:
37 float - if you include fann.h or floatfann.h
38 double - if you include doublefann.h
39 int - if you include fixedfann.h (please be aware that fixed point usage is
40 only to be used during execution, and not during training).
43 /* Enum: fann_train_enum
44 The Training algorithms used when training on <struct fann_train_data> with functions like
45 <fann_train_on_data> or <fann_train_on_file>. The incremental training looks alters the weights
46 after each time it is presented an input pattern, while batch only alters the weights once after
47 it has been presented to all the patterns.
49 FANN_TRAIN_INCREMENTAL - Standard backpropagation algorithm, where the weights are
50 updated after each training pattern. This means that the weights are updated many
51 times during a single epoch. For this reason some problems, will train very fast with
52 this algorithm, while other more advanced problems will not train very well.
53 FANN_TRAIN_BATCH - Standard backpropagation algorithm, where the weights are updated after
54 calculating the mean square error for the whole training set. This means that the weights
55 are only updated once during a epoch. For this reason some problems, will train slower with
56 this algorithm. But since the mean square error is calculated more correctly than in
57 incremental training, some problems will reach a better solutions with this algorithm.
58 FANN_TRAIN_RPROP - A more advanced batch training algorithm which achieves good results
59 for many problems. The RPROP training algorithm is adaptive, and does therefore not
60 use the learning_rate. Some other parameters can however be set to change the way the
61 RPROP algorithm works, but it is only recommended for users with insight in how the RPROP
62 training algorithm works. The RPROP training algorithm is described by
63 [Riedmiller and Braun, 1993], but the actual learning algorithm used here is the
64 iRPROP- training algorithm which is described by [Igel and Husken, 2000] which
65 is an variety of the standard RPROP training algorithm.
66 FANN_TRAIN_QUICKPROP - A more advanced batch training algorithm which achieves good results
67 for many problems. The quickprop training algorithm uses the learning_rate parameter
68 along with other more advanced parameters, but it is only recommended to change these
69 advanced parameters, for users with insight in how the quickprop training algorithm works.
70 The quickprop training algorithm is described by [Fahlman, 1988].
73 <fann_set_training_algorithm>, <fann_get_training_algorithm>
77 FANN_TRAIN_INCREMENTAL = 0,
83 /* Constant: FANN_TRAIN_NAMES
85 Constant array consisting of the names for the training algorithms, so that the name of an
86 training function can be received by:
88 char *name = FANN_TRAIN_NAMES[train_function];
94 static char const *const FANN_TRAIN_NAMES[] = {
95 "FANN_TRAIN_INCREMENTAL",
98 "FANN_TRAIN_QUICKPROP"
101 /* Enums: fann_activationfunc_enum
103 The activation functions used for the neurons during training. The activation functions
104 can either be defined for a group of neurons by <fann_set_activation_function_hidden> and
105 <fann_set_activation_function_output> or it can be defined for a single neuron by <fann_set_activation_function>.
107 The steepness of an activation function is defined in the same way by
108 <fann_set_activation_steepness_hidden>, <fann_set_activation_steepness_output> and <fann_set_activation_steepness>.
110 The functions are described with functions where:
111 * x is the input to the activation function,
113 * s is the steepness and
114 * d is the derivation.
116 FANN_LINEAR - Linear activation function.
117 * span: -inf < y < inf
119 * Can NOT be used in fixed point.
121 FANN_THRESHOLD - Threshold activation function.
122 * x < 0 -> y = 0, x >= 0 -> y = 1
123 * Can NOT be used during training.
125 FANN_THRESHOLD_SYMMETRIC - Threshold activation function.
126 * x < 0 -> y = 0, x >= 0 -> y = 1
127 * Can NOT be used during training.
129 FANN_SIGMOID - Sigmoid activation function.
130 * One of the most used activation functions.
132 * y = 1/(1 + exp(-2*s*x))
135 FANN_SIGMOID_STEPWISE - Stepwise linear approximation to sigmoid.
136 * Faster than sigmoid but a bit less precise.
138 FANN_SIGMOID_SYMMETRIC - Symmetric sigmoid activation function, aka. tanh.
139 * One of the most used activation functions.
141 * y = tanh(s*x) = 2/(1 + exp(-2*s*x)) - 1
144 FANN_SIGMOID_SYMMETRIC - Stepwise linear approximation to symmetric sigmoid.
145 * Faster than symmetric sigmoid but a bit less precise.
147 FANN_GAUSSIAN - Gaussian activation function.
148 * 0 when x = -inf, 1 when x = 0 and 0 when x = inf
153 FANN_GAUSSIAN_SYMMETRIC - Symmetric gaussian activation function.
154 * -1 when x = -inf, 1 when x = 0 and 0 when x = inf
156 * y = exp(-x*s*x*s)*2-1
159 FANN_ELLIOT - Fast (sigmoid like) activation function defined by David Elliott
161 * y = ((x*s) / 2) / (1 + |x*s|) + 0.5
162 * d = s*1/(2*(1+|x*s|)*(1+|x*s|))
164 FANN_ELLIOT_SYMMETRIC - Fast (symmetric sigmoid like) activation function defined by David Elliott
166 * y = (x*s) / (1 + |x*s|)
167 * d = s*1/((1+|x*s|)*(1+|x*s|))
169 FANN_LINEAR_PIECE - Bounded linear activation function.
173 FANN_LINEAR_PIECE_SYMMETRIC - Bounded Linear activation function.
178 <fann_set_activation_function_hidden>,
179 <fann_set_activation_function_output>
181 enum fann_activationfunc_enum
185 FANN_THRESHOLD_SYMMETRIC,
187 FANN_SIGMOID_STEPWISE,
188 FANN_SIGMOID_SYMMETRIC,
189 FANN_SIGMOID_SYMMETRIC_STEPWISE,
191 FANN_GAUSSIAN_SYMMETRIC,
192 /* Stepwise linear approximation to gaussian.
193 * Faster than gaussian but a bit less precise.
194 * NOT implemented yet.
196 FANN_GAUSSIAN_STEPWISE,
198 FANN_ELLIOT_SYMMETRIC,
200 FANN_LINEAR_PIECE_SYMMETRIC
203 /* Constant: FANN_ACTIVATIONFUNC_NAMES
205 Constant array consisting of the names for the activation function, so that the name of an
206 activation function can be received by:
208 char *name = FANN_ACTIVATIONFUNC_NAMES[activation_function];
212 <fann_activationfunc_enum>
214 static char const *const FANN_ACTIVATIONFUNC_NAMES[] = {
217 "FANN_THRESHOLD_SYMMETRIC",
219 "FANN_SIGMOID_STEPWISE",
220 "FANN_SIGMOID_SYMMETRIC",
221 "FANN_SIGMOID_SYMMETRIC_STEPWISE",
223 "FANN_GAUSSIAN_SYMMETRIC",
224 "FANN_GAUSSIAN_STEPWISE",
226 "FANN_ELLIOT_SYMMETRIC",
228 "FANN_LINEAR_PIECE_SYMMETRIC"
231 /* Enum: fann_errorfunc_enum
232 Error function used during training.
234 FANN_ERRORFUNC_LINEAR - Standard linear error function.
235 FANN_ERRORFUNC_TANH - Tanh error function, usually better
236 but can require a lower learning rate. This error function agressively targets outputs that
237 differ much from the desired, while not targetting outputs that only differ a little that much.
238 This activation function is not recommended for cascade training and incremental training.
241 <fann_set_train_error_function>, <fann_get_train_error_function>
243 enum fann_errorfunc_enum
245 FANN_ERRORFUNC_LINEAR = 0,
249 /* Constant: FANN_ERRORFUNC_NAMES
251 Constant array consisting of the names for the training error functions, so that the name of an
252 error function can be received by:
254 char *name = FANN_ERRORFUNC_NAMES[error_function];
258 <fann_errorfunc_enum>
260 static char const *const FANN_ERRORFUNC_NAMES[] = {
261 "FANN_ERRORFUNC_LINEAR",
262 "FANN_ERRORFUNC_TANH"
265 /* Enum: fann_stopfunc_enum
266 Stop criteria used during training.
268 FANN_STOPFUNC_MSE - Stop criteria is Mean Square Error (MSE) value.
269 FANN_STOPFUNC_BIT - Stop criteria is number of bits that fail. The number of bits; means the
270 number of output neurons which differ more than the bit fail limit
271 (see <fann_get_bit_fail_limit>, <fann_set_bit_fail_limit>).
272 The bits are counted in all of the training data, so this number can be higher than
273 the number of training data.
276 <fann_set_train_stop_function>, <fann_get_train_stop_function>
278 enum fann_stopfunc_enum
280 FANN_STOPFUNC_MSE = 0,
284 /* Constant: FANN_STOPFUNC_NAMES
286 Constant array consisting of the names for the training stop functions, so that the name of a
287 stop function can be received by:
289 char *name = FANN_STOPFUNC_NAMES[stop_function];
295 static char const *const FANN_STOPFUNC_NAMES[] = {
300 /* forward declarations for use with the callback */
302 struct fann_train_data;
303 /* Type: fann_callback_type
304 This callback function can be called during training when using <fann_train_on_data>,
305 <fann_train_on_file> or <fann_cascade_train>.
307 >typedef int (FANN_API * fann_callback_type) (struct fann *ann, struct fann_train_data *train,
308 > unsigned int max_epochs,
309 > unsigned int epochs_between_reports,
310 > float desired_error, unsigned int epochs);
312 The callback can be set by using <fann_set_callback> and is very usefull for doing custom
313 things during training. It is recommended to use this function when implementing custom
314 training procedures, or when visualizing the training in a GUI etc. The parameters which the
315 callback function takes is the parameters given to the <fann_train_on_data>, plus an epochs
316 parameter which tells how many epochs the training have taken so far.
318 The callback function should return an integer, if the callback function returns -1, the training
321 Example of a callback function:
322 >int FANN_API test_callback(struct fann *ann, struct fann_train_data *train,
323 > unsigned int max_epochs, unsigned int epochs_between_reports,
324 > float desired_error, unsigned int epochs)
326 > printf("Epochs %8d. MSE: %.5f. Desired-MSE: %.5f\n", epochs, fann_get_MSE(ann), desired_error);
331 <fann_set_callback>, <fann_train_on_data>
333 FANN_EXTERNAL typedef int (FANN_API * fann_callback_type) (struct fann *ann, struct fann_train_data *train,
334 unsigned int max_epochs,
335 unsigned int epochs_between_reports,
336 float desired_error, unsigned int epochs);
339 /* ----- Data structures -----
340 * No data within these structures should be altered directly by the user.
345 /* Index to the first and last connection
346 * (actually the last is a past end index)
348 unsigned int first_con;
349 unsigned int last_con;
350 /* The sum of the inputs multiplied with the weights */
352 /* The value of the activation function applied to the sum */
354 /* The steepness of the activation function */
355 fann_type activation_steepness;
356 /* Used to choose which activation function to use */
357 enum fann_activationfunc_enum activation_function;
359 } __attribute__ ((packed));
364 /* A single layer in the neural network.
368 /* A pointer to the first neuron in the layer
369 * When allocated, all the neurons in all the layers are actually
370 * in one long array, this is because we wan't to easily clear all
371 * the neurons at once.
373 struct fann_neuron *first_neuron;
375 /* A pointer to the neuron past the last neuron in the layer */
376 /* the number of neurons is last_neuron - first_neuron */
377 struct fann_neuron *last_neuron;
380 /* Struct: struct fann_error
382 Structure used to store error-related information, both
383 <struct fann> and <struct fann_train_data> can be casted to this type.
386 <fann_set_error_log>, <fann_get_errno>
390 enum fann_errno_enum errno_f;
396 /* Struct: struct fann
397 The fast artificial neural network(fann) structure.
399 Data within this structure should never be accessed directly, but only by using the
400 *fann_get_...* and *fann_set_...* functions.
402 The fann structure is created using one of the *fann_create_...* functions and each of
403 the functions which operates on the structure takes *struct fann * ann* as the first parameter.
406 <fann_create_standard>, <fann_destroy>
410 /* The type of error that last occured. */
411 enum fann_errno_enum errno_f;
413 /* Where to log error messages. */
416 /* A string representation of the last error. */
419 /* the learning rate of the network */
422 /* The learning momentum used for backpropagation algorithm. */
423 float learning_momentum;
425 /* the connection rate of the network
426 * between 0 and 1, 1 meaning fully connected
428 float connection_rate;
430 /* is 1 if shortcut connections are used in the ann otherwise 0
431 * Shortcut connections are connections that skip layers.
432 * A fully connected ann with shortcut connections are a ann where
433 * neurons have connections to all neurons in all later layers.
435 unsigned int shortcut_connections;
437 /* pointer to the first layer (input layer) in an array af all the layers,
438 * including the input and outputlayers
440 struct fann_layer *first_layer;
442 /* pointer to the layer past the last layer in an array af all the layers,
443 * including the input and outputlayers
445 struct fann_layer *last_layer;
447 /* Total number of neurons.
448 * very usefull, because the actual neurons are allocated in one long array
450 unsigned int total_neurons;
452 /* Number of input neurons (not calculating bias) */
453 unsigned int num_input;
455 /* Number of output neurons (not calculating bias) */
456 unsigned int num_output;
458 /* The weight array */
461 /* The connection array */
462 struct fann_neuron **connections;
464 /* Used to contain the errors used during training
465 * Is allocated during first training session,
466 * which means that if we do not train, it is never allocated.
468 fann_type *train_errors;
470 /* Training algorithm used when calling fann_train_on_..
472 enum fann_train_enum training_algorithm;
475 /* the decimal_point, used for shifting the fix point
476 * in fixed point integer operatons.
478 unsigned int decimal_point;
480 /* the multiplier, used for multiplying the fix point
481 * in fixed point integer operatons.
482 * Only used in special cases, since the decimal_point is much faster.
484 unsigned int multiplier;
486 /* When in choosen (or in fixed point), the sigmoid function is
487 * calculated as a stepwise linear function. In the
488 * activation_results array, the result is saved, and in the
489 * two values arrays, the values that gives the results are saved.
491 fann_type sigmoid_results[6];
492 fann_type sigmoid_values[6];
493 fann_type sigmoid_symmetric_results[6];
494 fann_type sigmoid_symmetric_values[6];
497 /* Total number of connections.
498 * very usefull, because the actual connections
499 * are allocated in one long array
501 unsigned int total_connections;
503 /* used to store outputs in */
506 /* the number of data used to calculate the mean square error.
508 unsigned int num_MSE;
510 /* the total error value.
511 * the real mean square error is MSE_value/num_MSE
515 /* The number of outputs which would fail (only valid for classification problems)
517 unsigned int num_bit_fail;
519 /* The maximum difference between the actual output and the expected output
520 * which is accepted when counting the bit fails.
521 * This difference is multiplied by two when dealing with symmetric activation functions,
522 * so that symmetric and not symmetric activation functions can use the same limit.
524 fann_type bit_fail_limit;
526 /* The error function used during training. (default FANN_ERRORFUNC_TANH)
528 enum fann_errorfunc_enum train_error_function;
530 /* The stop function used during training. (default FANN_STOPFUNC_MSE)
532 enum fann_stopfunc_enum train_stop_function;
534 /* The callback function used during training. (default NULL)
536 fann_callback_type callback;
538 /* Variables for use with Cascade Correlation */
540 /* The error must change by at least this
541 * fraction of its old value to count as a
542 * significant change.
544 float cascade_output_change_fraction;
546 /* No change in this number of epochs will cause
549 unsigned int cascade_output_stagnation_epochs;
551 /* The error must change by at least this
552 * fraction of its old value to count as a
553 * significant change.
555 float cascade_candidate_change_fraction;
557 /* No change in this number of epochs will cause
560 unsigned int cascade_candidate_stagnation_epochs;
562 /* The current best candidate, which will be installed.
564 unsigned int cascade_best_candidate;
566 /* The upper limit for a candidate score
568 fann_type cascade_candidate_limit;
570 /* Scale of copied candidate output weights
572 fann_type cascade_weight_multiplier;
574 /* Maximum epochs to train the output neurons during cascade training
576 unsigned int cascade_max_out_epochs;
578 /* Maximum epochs to train the candidate neurons during cascade training
580 unsigned int cascade_max_cand_epochs;
582 /* An array consisting of the activation functions used when doing
585 enum fann_activationfunc_enum *cascade_activation_functions;
587 /* The number of elements in the cascade_activation_functions array.
589 unsigned int cascade_activation_functions_count;
591 /* An array consisting of the steepnesses used during cascade training.
593 fann_type *cascade_activation_steepnesses;
595 /* The number of elements in the cascade_activation_steepnesses array.
597 unsigned int cascade_activation_steepnesses_count;
599 /* The number of candidates of each type that will be present.
600 * The actual number of candidates is then
601 * cascade_activation_functions_count *
602 * cascade_activation_steepnesses_count *
603 * cascade_num_candidate_groups
605 unsigned int cascade_num_candidate_groups;
607 /* An array consisting of the score of the individual candidates,
608 * which is used to decide which candidate is the best
610 fann_type *cascade_candidate_scores;
612 /* The number of allocated neurons during cascade correlation algorithms.
613 * This number might be higher than the actual number of neurons to avoid
614 * allocating new space too often.
616 unsigned int total_neurons_allocated;
618 /* The number of allocated connections during cascade correlation algorithms.
619 * This number might be higher than the actual number of neurons to avoid
620 * allocating new space too often.
622 unsigned int total_connections_allocated;
624 /* Variables for use with Quickprop training */
626 /* Decay is used to make the weights not go so high */
627 float quickprop_decay;
629 /* Mu is a factor used to increase and decrease the stepsize */
632 /* Variables for use with with RPROP training */
634 /* Tells how much the stepsize should increase during learning */
635 float rprop_increase_factor;
637 /* Tells how much the stepsize should decrease during learning */
638 float rprop_decrease_factor;
640 /* The minimum stepsize */
641 float rprop_delta_min;
643 /* The maximum stepsize */
644 float rprop_delta_max;
646 /* The initial stepsize */
647 float rprop_delta_zero;
649 /* Used to contain the slope errors used during batch training
650 * Is allocated during first training session,
651 * which means that if we do not train, it is never allocated.
653 fann_type *train_slopes;
655 /* The previous step taken by the quickprop/rprop procedures.
656 * Not allocated if not used.
658 fann_type *prev_steps;
660 /* The slope values used by the quickprop/rprop procedures.
661 * Not allocated if not used.
663 fann_type *prev_train_slopes;
665 /* The last delta applied to a connection weight.
666 * This is used for the momentum term in the backpropagation algorithm.
667 * Not allocated if not used.
669 fann_type *prev_weights_deltas;