/*
Optimizer.h
Written by Matthew Fisher

Generic non-linear optimizer.  Uses conjugate gradient descent (or other inferior/sub-component algorithms like line search.)

To use the class, allocate the member data Variables to contain the number of non-linear variables present.
Then set each variable's Target paramater to point to the non-linear values that the function will change
in an attempt to minimize the Error function which must also be set.  The best fit will automatically be
stored in the variable targets.  I don't thoroughly describe all the types of learning because that would take
a very long time.  The best is conjugate gradient descent, and to use it you need to call ConjugateReset first
(a repeat rate of 20 is a good number.)
*/

struct WeightElement {
    double *Target;     //pointer to variable to optimize
    double Partial,     //partial derivative
        OldPartial,     //previous partial derivative
        LineSearchStart,        //start for line search
        LineSearchDirection;    //direction for line search
};

class Optimizer {
public:
    //Inferior Optimization schemes
    void GradientDescent(double LearningRate);                              //gradient descent learning
    void MomentumGradientDescent(double LearningRate, double MomentumRate); //gradient descent learning with momentum
    void SteepestDescent();                                                 //steepest descent learning
    void AdaptiveLearning(double &LearningRate, double Alpha, double Beta); //adaptive learning

    //Conjugate Gradient
    void ConjugateReset(int _RepeatRate);   //resets the conjugate gradient algorithm and sets it up to reset
                                            //again every RepeatRate steps.
    void ConjugateGradient();               //conjugate gradient descent step

    int RepeatRate, RepeatsLeft;        //conjugate gradient paramaters
    Vector<WeightElement> Variables;    //the array of variables to optimize
    double (*Error)();                  //the function to minimize

private:
    //the following are internal functions used by the learning algorithms above
    void LineSearch();          //line search learning
    void ComputeGradient();     //computes the gradient of the error (no learning)
};