java help (warning: very hard to follow code!!)

witnessmenow

Hi all,

Sorry in advance for the horrible coding ! I'm not the greatest at this whole java malarky!

Have a project due this week on reinforcement learning. The object is for the program to learn the best solution to "the towers of hanoi" on its own. I'm almost positive I have it going other than a few hiccups.

I let the game play through randomly to determine all the possible states and possible moves from each state.

The problems:
when i select learn mode at the start it makes the right move most of the time but sometimes it makes illegal moves. The moves are no longer based on my original game code, its relying on the previously populated arrays. It should work, and the fact thats its working most of the time indicates i'm close( i hope). If you run the qArray choice (this runs trhought the code randomly populating the Qarray) and then choose "move test" it will make it plays through in the least amount of moves with no illegal moves so my problem must be with choice 5

writing to file, code is basically straight from a lecture, cant figure out why its not writing?

Note:
to run the "learn" part you need to first run intialise arrays to populate the arrays.

Here's the code: (i've marked parts that are definitly working in purple) :

import javax.swing.JOptionPane;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.PrintWriter;
import java.util.*;

//Brian L
//17/12/07

public class SelfTower 
{
	public static void main( String[] args ) throws IOException 
	{
		int towerSelect = 0,
			destination = 0,
			addressCount = 0,
			tempBlock = 0,
			tempLocation = 0,
		    numBlocks = 0,
		    testCompleted = 0,
			completedCheck = 0,
			mostSignificant = 0,
			mostSignificantValue = 0,
			intialise = 0,
			moveCount = 0,
			addressKey = 0,
			previousKey = 0,
			addressDecode = 0,
			preferedMove = 0,
			preferedAddress = 0,
			tempCount,
			reward = 0,
			temp =0,
			explore = 0,
			previousDecode = 0,
			timesCompleted = 0;
		int[] addressArray = new int[14] ;
		int[][] rewardArray = new int[14][14] ;
		int[][] qArray = new int[14][14] ;
		int[][] loadTest = new int[14][];
		int[][] actionArray = new int[14][];
		int[][] tempArray = new int[14][5];
		String tempStr;
		String rewardFile = "rewardArray.txt";
		String addressFile = "addressArray.txt";
		String moveFile = "moveArray.txt";
		String qFile = "qArray.txt";
		boolean check = false;
		boolean stage1 = false;
		boolean moveCheck = false;
		boolean towerCheck = false;
		boolean checkArray = false;
		boolean completed = false;
		boolean nonZero = false;
		boolean moving = false;
		boolean skip = false;
		Random generator = new Random();
		/*
		while (check == false)
		{
			//Selecting the number of blocks to play with
			tempStr = JOptionPane.showInputDialog("Please enter number of blocks (3-8)");
			numBlocks = Integer.parseInt(tempStr);
			
			if ((numBlocks>=3)&&(numBlocks<=8))
			{
				check = true;	
			}
			else
			{
				JOptionPane.showMessageDialog(null, "Incorrect selection");
			}
		}
		*/
		
		
		numBlocks=3;
		FileWriter fw= new FileWriter("progress Data.txt");
		PrintWriter pw = new PrintWriter(fw);
		//Initialise reward table
		
		tempStr = JOptionPane.showInputDialog("Please make a selection: \n1: Intialise arrays, \n2:Reward Test, \n3:QArray Test, \n4:Move Test, \n5:learn \n");
		intialise = Integer.parseInt(tempStr);
		
		if (intialise == 5)
		{
			
			while(check == false)
			{
				tempStr = JOptionPane.showInputDialog("Please enter the precentage that you would like the agent to make an exploritary move(0-100):");
				explore = Integer.parseInt(tempStr);
				if (explore >= 0 && explore <=100)
				{
					check = true;
				}
				else
				{
					JOptionPane.showMessageDialog(null, "Invalid selection");
				}
			}

		}
		
		if (intialise == 1)
		{
			for ( int i = 0; i < 14; i++ )
				{
					for ( int j = 0; j < 14; j++ )
					{
						
						rewardArray[i][j] = -1;
						
					}
					
				}
			
			rewardArray[12][13] = 1000;
			rewardArray[13][13] = 1000;
			
			for ( int i = 0; i < 14; i++ )
			{
				for ( int j = 0; j < 14; j++ )
				{
					
					qArray[i][j] = 0;
					
				}
				
			}
			
			save2dArray(qFile, qArray);
		}
		else
		{
			rewardArray = load2dArray(rewardFile);
			actionArray = load2dArray(moveFile);
			addressArray = load1dArray(addressFile);
			qArray = load2dArray(qFile);
		}
		
		int gamma = 8;
		
		
		
		//making an array using the previous selection
		int[][] game = new int[numBlocks][3] ;
		
		//Setting up first tower 
		for ( int i = 0; i < numBlocks; i++ )
		{
			game[i][0]= (i + 1);
			completedCheck = completedCheck + (i+1);
		}
		
		while(completed == false)
		{
			//Printing the tower
			skip = false;
			for ( int i = 0; i < numBlocks; i++ )
			{
				for ( int j = 0; j < 3; j++ )
				{
					System.out.print(" ");
					System.out.print( game[i][j] );
					System.out.print(" ");
				}
				System.out.println();
			}
			System.out.println("---------");
			
			mostSignificant = 0;
			mostSignificantValue = game[2][0];
			//addressKey = 0;
			
			if ( game[2][1] > game[2][0])
			{
				mostSignificant = 1;
				mostSignificantValue = game[2][1];
			}
			System.out.println("MS = " + mostSignificant);
			
			//Getting address key
			if (moveCount > 1)
			{
				previousKey = addressKey;
				previousDecode = decodeAddress(previousKey, addressArray);
			}
			addressKey = getAddressKey(mostSignificant, game);
			
			System.out.println("Address key = " + addressKey);
			System.out.println("Previous key = " + previousKey);
			
			addressDecode = decodeAddress(addressKey, addressArray);
			
			
			if(intialise == 2 || intialise == 4 || intialise == 5)
			{
				reward = reward + rewardArray[previousDecode][addressDecode];
			}
			
			if (intialise == 3 || intialise == 4 || intialise == 5)
			{
				qArray[previousDecode][addressDecode] = getQValue(gamma, addressDecode, previousDecode, rewardArray, actionArray, qArray);	

			}
			
			
			
			if (stage1 == true)
			{
				System.out.println("Address decode = " + addressDecode);
				System.out.println("Previous decode = " + previousDecode);
				
				moveCheck = false;
				for ( int i = 0; i < 5; i++ )
				{
					if (addressDecode == tempArray[previousDecode][i])
					{
						i = 6;
						moveCheck = true;
						
					}
				}
				
				if (moveCheck == false)
				{
					for ( int i = 1; i < 5; i++ )
					{
						if (tempArray[previousDecode][i] == previousDecode)
						{
							tempArray[previousDecode][i] = addressDecode;
							i = 6;
						}
					}
				}
			}
			
			
			
			//Populating address array 
			checkArray = false;
			for ( int i = 0; i < 14; i++ )
			{
				if (addressKey == addressArray[i])
				{
					checkArray = true;
				}
			}
			if (checkArray == false)
			{
				//populating the address array
				for ( int i = 0; i < 14; i++ )
				{
					if (addressArray[i] == 0)
					{
						addressArray[i] = addressKey;
						i = 20;
					}
				}
			}
			
			//checking for completion
		
			testCompleted=0;
			if (intialise == 4 || intialise == 5)
			{
				for ( int i = 0; i < numBlocks; i++ )
				{
					testCompleted = game[i][2]+ testCompleted;
				}
			}
			if (testCompleted == completedCheck && intialise == 4)
			{
				JOptionPane.showMessageDialog(null, "Completed. It took "+moveCount+" moves");
				System.out.println("Reward = " + reward);
				completed=true;
			}	
			if (testCompleted == completedCheck && intialise == 5)
			{
				String tempString = String.valueOf(reward);
				pw.println(tempString);
				timesCompleted = timesCompleted +1;
				for ( int i = 0; i < numBlocks; i++ )
				{
					for ( int j = 0; j < 3; j++ )
					{
						
						game[i][j] = 0;
						
					}
					
				}
				
				for ( int i = 0; i < numBlocks; i++ )
				{
					game[i][0]= (i + 1);					
				}
				reward = 0;
				moveCount = 0;
				mostSignificant = 0;
				addressKey = getAddressKey(mostSignificant, game);
				addressDecode = decodeAddress(addressKey, addressArray);
				skip =true;
			}
			
			if (timesCompleted>10)
			{
				
				
				JOptionPane.showMessageDialog(null, "Completed the game 10 times");
			
				completed=true;
				
			}
			
			
			addressCount = 0;
			for ( int i = 0; i < 14; i++ )
			{
				if (addressArray[i]!=0)
				{
						
					addressCount= addressCount +1;
				
				}
			
				
			}
			
			System.out.println("Address count = " + addressCount);
			//intiate
			if (stage1 == false)
			{
				if (addressCount == 14 && intialise == 1)
				{
					Arrays.sort(addressArray);
					save1dArray (addressFile, addressArray);
					save2dArray (rewardFile, rewardArray);
					JOptionPane.showMessageDialog(null, "Intiation stage 1 complete, please press ok to continue to stage 2");
					stage1 = true;
					
					for ( int i = 0; i < 14; i++ )
					{
						for ( int j = 0; j < 5; j++ )
						{
							tempArray [i][j] = i;
						}
							
						
					}
						
				}
			}
			if (moveCount == 10000 && intialise == 1)
			{
				JOptionPane.showMessageDialog(null, "Intiation stage 2 complete, please restart program");
				
				completed = true;
				

				for ( int i = 0; i < 14; i++ )
				{
					tempCount = 0;
					for ( int j = 1; j < 5; j++ )
					{
							
						if (tempArray[i][j] != i )
						{
							tempCount = tempCount +1;
						}
							
					}
					
					actionArray[i]= new int[tempCount+1];
					
					
				}
				
				for (int i = 0; i < actionArray.length; i++)
				{
					
		            for (int j = 0; j < actionArray[i].length; j++) 
		            {
		            	
		                actionArray[i][j] = tempArray[i][j];
		            
		            }
		            
		            
		        }
				
				save2dArray (moveFile, actionArray);
				/*
				loadTest = load2dArray(moveFile);
				
				for (int i = 0; i < loadTest.length; i++)
				{
					
		            for (int j = 0; j < loadTest[i].length; j++) 
		            {
		            	
		               System.out.print(loadTest[i][j] + " ");
		            
		            }
		            
		            System.out.println();
		        }
		        */
				
			}
			if(moveCount == 10000 && intialise == 3)
			{
				completed = true;
				save2dArray(qFile, qArray);
				
				
				for (int i = 0; i < qArray.length; i++)
				{
				
					for (int j = 0; j < qArray[i].length; j++) 
					{
						if(qArray[i][j]<0)
						{
							System.out.print(qArray[i][j] + "    ");
						}
						else
						{
							System.out.print(" " + qArray[i][j] + "    ");
						}
	            
					}
	            
					System.out.println();
				}
			}
			
			if ( intialise == 4)
			{
				explore =0;
				preferedMove = getPreferedMove(addressDecode, explore, actionArray , qArray);
				moveCount= moveCount+1;
				game = performMove(mostSignificant, preferedMove, mostSignificantValue, preferedAddress, game , addressArray);
			
				}
			
			
			if (intialise == 5 && skip == false)
			{
				preferedMove = getPreferedMove(addressDecode, explore, actionArray , qArray);
				moveCount= moveCount+1;
				game = performMove(mostSignificant, preferedMove, mostSignificantValue, preferedAddress, game , addressArray);
			}
	        
			
		
				
				
				
			
			
	[COLOR="Magenta"]		//Main game handling
			if (completed==false && intialise != 4)
			{
				moving = false;
				while(moving == false)
				{
					towerCheck=false;
					while(towerCheck == false)
					{
						//selecting source tower
						int randomIndex = generator.nextInt( 3 );
						towerSelect = randomIndex +1 ;
						if ((towerSelect>=1)&&(towerSelect<=3))//making sure selection is in bounds
						{
							for ( int i = numBlocks; i > 0; i-- )
							{
								if (game[i-1][(towerSelect-1)] != 0)
								{
									
									tempLocation = i-1;
									tempBlock = (game[i-1][(towerSelect-1)]);
									towerCheck = true;
								}
							}
							if (towerCheck==false)
							{
								//JOptionPane.showMessageDialog(null, "there is no blocks on this tower");
							}
						}
						else
						{
							//JOptionPane.showMessageDialog(null, "Invalid selection");
						}
					}
					int randomIndex = generator.nextInt( 3 );
					destination = randomIndex+1;
					if ((destination>=1)&&(destination<=3))
					{
						if (towerSelect!=destination)
						{
							nonZero=false;
							for ( int i = 0; i < numBlocks; i++ )
							{
								if (game[i][(destination-1)] != 0)
								{
									nonZero=true;
									if ((game[i][(destination-1)])>tempBlock)
									{
										(game[i-1][(destination-1)])=tempBlock;
										(game[tempLocation][towerSelect-1])=0;
										moving=true;
										moveCount = moveCount + 1;
									}
									else
									{
										//JOptionPane.showMessageDialog(null, "Selected block is bigger than destination block");
									}
									i=numBlocks;
								}
								
							}
							if (nonZero == false)
							{
								game[numBlocks-1][destination-1]=tempBlock;
								game[tempLocation][towerSelect-1]=0;
								moving=true;
								moveCount = moveCount + 1;
							}
						}
						else
						{
							//Exiting the loop if destination and select are the same
							moving=true;
							moveCount = moveCount + 1;
						}
										
					}
				}
				
				
			}
		}
			
		
	
	
		System.exit(0);	
}
	  public static void save2dArray(String filename, int[][] output_veld)
	  {
		  try 
		  {
			  FileOutputStream fos = new FileOutputStream(filename);
		       
			  ObjectOutputStream out = new ObjectOutputStream(fos);
			  out.writeObject(output_veld);
			  out.flush();
			  out.close();
		  }
		  catch (IOException e) 
		  {
			  System.out.println(e); 
		  }
	  }
	  public static int[][] load2dArray(String filename) {
	      try {
	        FileInputStream fis = new FileInputStream(filename);
	        ObjectInputStream in = new ObjectInputStream(fis);
	        int[][] gelezen_veld = (int[][])in.readObject();
	        in.close();
	        return gelezen_veld;
	      }
	      catch (Exception e) {
	          System.out.println(e);
	      }
	      return null;
	  }
	  
	  public static int getAddressKey(int MSP, int[][] tempGame) 
	  {
	      int addressKey = 0;
		  
		  addressKey = addressKey + (tempGame[0][2])*100000;
		  addressKey = addressKey + (tempGame[1][2])*10000;
		  addressKey = addressKey + (tempGame[2][2])*1000;
	
		  //MSO-pole code

		  addressKey = addressKey + (tempGame[0][MSP])*100;
		  addressKey = addressKey + (tempGame[1][MSP])*10;
		  addressKey = addressKey + (tempGame[2][MSP]);
		  
		  return addressKey;
	      
	  }
	  
	  public static void save1dArray(String filename, int[] output_veld)
	  {
		  try 
		  {
			  FileOutputStream fos = new FileOutputStream(filename);
		       
			  ObjectOutputStream out = new ObjectOutputStream(fos);
			  out.writeObject(output_veld);
			  out.flush();
			  out.close();
		  }
		  catch (IOException e) 
		  {
			  System.out.println(e); 
		  }
	  }
	  public static int[] load1dArray(String filename) {
	      try {
	        FileInputStream fis = new FileInputStream(filename);
	        ObjectInputStream in = new ObjectInputStream(fis);
	        int[] gelezen_veld = (int[])in.readObject();
	        in.close();
	        return gelezen_veld;
	      }
	      catch (Exception e) {
	          System.out.println(e);
	      }
	      return null;
	  }
	  
	  public static int decodeAddress(int address, int[] tempAddressArray) 
	  {
	      int addressDec = 0;
	      
	  	for ( int j = 0; j < 14; j++ )
		{
			if (address == tempAddressArray[j])
			{
				addressDec = j;
			}
				
		}
		  
		  return addressDec;
	      
	  }[/COLOR]
	  
	  public static int getQValue(int gamma,int addressDecode, int previousDecode, int[][] rewardArray, int[][]actionArray , int[][]qArray) 
	  {
	      int q,
	      largest;
	      
	      
			q = 0;
			largest = qArray[addressDecode][actionArray[addressDecode][0]];

			for (int j = 1; j < actionArray[addressDecode].length; j++)
			{
				if (qArray[addressDecode][actionArray[addressDecode][j]]>largest)
				{
					largest = qArray[addressDecode][actionArray[addressDecode][j]];
				}
			
			}
			q = rewardArray[previousDecode][addressDecode] + gamma*largest;
			q = q/10;
			
			return q;
				
	  }
	  
	  public static int getPreferedMove(int addressDecode, int explore, int[][]actionArray , int[][]qArray) 
	  {
		  	Random generator = new Random();
			int randomIndex1 = generator.nextInt(100);
			int randomIndex2 = generator.nextInt(actionArray[addressDecode].length);
			int preferedMove = 0;
			
			if (randomIndex1+1 > explore)
			{
				boolean multibleLargest = false;
				int largest = 0;
				
				
				largest = qArray[addressDecode][actionArray[addressDecode][0]];
				for (int j = 1; j < actionArray[addressDecode].length; j++)
				{
					if (largest == qArray[addressDecode][actionArray[addressDecode][j]])
					{
						multibleLargest = true;
						
					}
					if (qArray[addressDecode][actionArray[addressDecode][j]]>largest)
					{
						largest = qArray[addressDecode][actionArray[addressDecode][j]];
						preferedMove = actionArray[addressDecode][j];
						multibleLargest = false;
					}
				}
				
				while (multibleLargest == true)
				{
					randomIndex2 = generator.nextInt( actionArray[addressDecode].length );
					if (largest == qArray[addressDecode][actionArray[addressDecode][randomIndex2]])
					{
						preferedMove = actionArray[addressDecode][randomIndex2];
						multibleLargest = false;
					}
							
				}
				
			}
			else
			{
				System.out.println("random");
				
					randomIndex2 = generator.nextInt( actionArray[addressDecode].length );
					preferedMove = actionArray[addressDecode][randomIndex2];
			
			}
			
			return preferedMove;
	  }
	  public static int [][] performMove(int mostSignificant, int preferedMove, int mostSignificantValue, int preferedAddress, int[][]game , int[]addressArray) 
	  {
			boolean threePresent = false;
			boolean twoPresent = false;
			boolean onePresent = false;
			
			
			for ( int i = 0; i < game.length; i++ )
			{
				for ( int j = 0; j < 3; j++ )
				{
					game[i][j] = 0;
				}
				
			}
			
			
			
			
			preferedAddress = addressArray[preferedMove];
			
			game[0][2] = preferedAddress/100000;
			if(mostSignificantValue == game[0][2])
			{
				mostSignificant = 1 - mostSignificant;
			}
			preferedAddress = preferedAddress - game[0][2]*100000;
			game[1][2] = preferedAddress/10000;
			if(mostSignificantValue == game[1][2])
			{
				mostSignificant = 1 - mostSignificant;
			}
			preferedAddress = preferedAddress - game[1][2]*10000;
			game[2][2] = preferedAddress/1000;
			if(mostSignificantValue == game[2][2])
			{
				mostSignificant = 1 - mostSignificant;
			}
			preferedAddress = preferedAddress - game[2][2]*1000;

		
			  //MSO-pole code
			
			if(mostSignificantValue < (preferedAddress%10))
			{
				mostSignificant = 1 - mostSignificant;
			}

			game[0][mostSignificant] = preferedAddress/100;
			preferedAddress = preferedAddress - game[0][mostSignificant]*100;
			game[1][mostSignificant] = preferedAddress/10;
			preferedAddress = preferedAddress - game[1][mostSignificant]*10;
			game[2][mostSignificant] = preferedAddress;
			
			for ( int i = 0; i < game.length; i++ )
			{
				for ( int j = 0; j < 3; j++ )
				{
					if ( game[i][j] == 3)
					{
						threePresent = true;
					}
					if ( game[i][j] == 2)
					{
						twoPresent = true;
					}
					if ( game[i][j] == 1)
					{
						onePresent = true;
					}
				}
				
			}
			int temp1 = 2;
			if ( threePresent == false )
			{
				game[temp1][(1-mostSignificant)] = 3;
				temp1 = temp1 - 1;
			}
			if ( twoPresent == false )
			{
				game[temp1][(1-mostSignificant)] = 2;
				temp1 = temp1 - 1;
			}
			if ( onePresent == false )
			{
				game[temp1][(1-mostSignificant)] = 1;
				
			}
			
			return game;
	  }
	  
}

Thanks in advance for any help you can give, I'll really aprreciate it:)

Cantab.

Jaysus!

816 lines to solve the Towers of Hanoi!

witnessmenow

Ok solved one of them myself after picking through the code and testing everything!

little quiz for ya:

{
int test;
test = 023;
}

the curent value of test is...... you guessed it 19!

was obviously a mistake to assume 023 was the same as 23!

Dar

Indeed,

23 is decimal: 23
023 is octal: 19
0x23 is hexadecimal: 35