All Data Structures Files Functions Variables Enumerations Enumerator Properties Defines
/Projects/Cogito/src/Objects/QState.m
Go to the documentation of this file.
00001 //
00002 //  QState.m
00003 //  Author: Thomas Taylor
00004 //
00005 //  Basic class to hold info about a state in Q-learning
00006 //
00007 //  06/02/2012: Created class
00008 //
00009 
00010 #import "QState.h"
00011 
00012 @interface QState()
00013 
00014 -(CCArray*)getDataForAction:(Action)_action;
00015 
00016 @end
00017 
00018 @implementation QState
00019 
00020 #pragma mark -
00021 #pragma mark Initialisation
00022 
00027 -(id)initStateForObject:(GameObject*)_object withReward:(float)_reward
00028 {    
00029     self = [super initStateForObject:_object];
00030     
00031     if (self != nil) 
00032     {
00033         reward = _reward;        
00034     }
00035     return self;
00036 }
00037 
00038 #pragma mark -
00039 
00045 -(Action)getOptimumAction
00046 {
00047     Action optimumAction = -1;
00048     float maxQValue = -1000000;
00049     CCArray* actionsArray = [self getActions];
00050     
00051     for (int i = 0; i < [actionsArray count]; i++) 
00052     {
00053         float qValue = [self getQValueForAction:[[actionsArray objectAtIndex:i] intValue]];
00054         if(qValue > maxQValue) 
00055         {
00056             maxQValue = qValue;
00057             optimumAction = [[actionsArray objectAtIndex:i] intValue];
00058         }
00059     }
00060     
00061     return optimumAction;
00062 }
00063 
00064 #pragma mark -
00065 #pragma mark Q-Value Calculations
00066 
00072 -(float)calculateMaxQValue
00073 {    
00074     // set a very low starting value
00075     float maximumQValue = -1000000.0f;
00076     
00077     CCArray* actionsArray = [self getActions];
00078     
00079     for (int i = 0; i < [actionsArray count]; i++) 
00080     {
00081         Action action = [[actionsArray objectAtIndex:i] intValue];
00082         float tempQValue = [self getQValueForAction:action];
00083         if(tempQValue > maximumQValue) maximumQValue = tempQValue;
00084     }
00085     
00086     return maximumQValue;
00087 }
00088 
00094 -(float)getQValueForAction:(Action)_action
00095 {
00096     // throw any relevant error messages before continuing
00097     if([actions count] == 0) { CCLOG(@"%@.getQValueForAction: Error, actions not initialised", NSStringFromClass([self class])); return -1000000.0f; }
00098     else if([[actions objectAtIndex:0] count] < 2) { CCLOG(@"%@.getQValueForAction: Error, Q-values not initialised %i", NSStringFromClass([self class]), [actions count]); return -1000000.0f; }
00099     
00100     CCArray* qData = [self getDataForAction:_action];
00101     if (qData != nil) return [[qData objectAtIndex:1] floatValue];
00102     else 
00103     {
00104         CCLOG(@"%@.getQValueForAction: %@ action: %i", NSStringFromClass([self class]), [Utils getObjectAsString:gameObject.gameObjectType], _action);
00105         return -1000000.0f;
00106     }
00107 }
00108 
00114 -(void)setQValue:(float)_qValue forAction:(Action)_action
00115 {
00116     CCArray* qData = [self getDataForAction:_action];
00117     if (qData != nil)
00118     {   
00119         [qData removeObjectAtIndex: 1];
00120         [qData insertObject:[NSNumber numberWithFloat:_qValue] atIndex:1];
00121     }    
00122 }
00123 
00124 #pragma mark -
00125 
00131 -(CCArray*)getDataForAction:(Action)_action
00132 {    
00133     for (int i = 0; i < [actions count]; i++) 
00134     {
00135         Action tempAction = [[[actions objectAtIndex:i] objectAtIndex:0] intValue];
00136         if(tempAction == _action) return [actions objectAtIndex:i];
00137     }
00138     
00139     // shouldn't get here
00140     return nil;
00141 }
00142 
00147 -(CCArray*)getActions
00148 {    
00149     CCArray* tempArray = [CCArray arrayWithCapacity:[actions count]];
00150         
00151     for (int i = 0; i < [actions count]; i++) [tempArray addObject:[[actions objectAtIndex:i] objectAtIndex:0]];
00152         
00153     return tempArray;
00154 }
00155 
00161 -(void)setActions:(CCArray*)_actions
00162 {    
00163     if([actions count] > 0) return;
00164     
00165     for (int i = 0; i < [_actions count]; i++) 
00166         [actions addObject:[NSMutableArray arrayWithObjects:[_actions objectAtIndex:i],[NSNumber numberWithFloat:0.0f],nil]];
00167 }
00168 
00173 -(float)getReward
00174 {
00175     return reward;
00176 }
00177 
00178 @end