package org.occ.TileCache;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import javax.imageio.ImageIO;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.occ.utilities.GeoSpatialUtilities;
import com.bbn.openmap.proj.coords.BoundingBox;
/**
* BuildTileCache is a Hadoop MapReduce program for building a set of files that will
* be used in an OGC WMS
*
* @author alevine@texeltek.com
*
*/
public class BuildTileCacheFromSequence extends Configured implements Tool{
// input directory in HDFS
private static Path inputPath = null;
// output directory in HDFS
private static Path outputPath = null;
// zoom level string from the input
private static String zoomLevelsStr = null;
/**
* MapPictures is the class that does the mapping of source imagery to destination
* imagery.
*
* @author alevine@texeltek.com
*
*/
public static class MapPictures extends org.apache.hadoop.mapreduce.Mapper<Text, BytesWritable, Text, BytesWritable>{
int [] zoomLevels = null; // array of zoom levels to process
int width = 256; // width of output image
int height = 256; // height of output image
/**
* setup() is run before any mappers are run
*/
public void setup(Context context){
// get the configuration of the running system
Configuration conf = context.getConfiguration();
// get the zoom levels
zoomLevels = getZoomLevels(conf.get("zoomLevelsStr"));
// get the output image width
width = Integer.parseInt(conf.get("imageWidth"));
// get the output image height
height = Integer.parseInt(conf.get("imageHeight"));
} // end setup
/**
* map() is the mapper function for pictures
*
* as of 20100609 we are looking at Plate Carree only
*
*/
public void map(Text key, BytesWritable img, Context context) throws IOException, InterruptedException{
// source image bounding box from the key
BoundingBox srcBox = GeoSpatialUtilities.getBoundingBoxFromFileName(key.toString());
// get the bytes of the image from the value
byte[] imgBytes = img.getBytes();
// create a stream from the bytes
InputStream is = new ByteArrayInputStream(imgBytes);
// get the image from the bytes
BufferedImage srcImg = ImageIO.read(is);
// close the input stream
is.close();
// determine the source image vertical degree per pixel
double srcVertDPP = (srcBox.getMaxY() - srcBox.getMinY()) / ((double) srcImg.getHeight());
// determine the source image horizontal degree per pixel
double srcHorzDPP = (srcBox.getMaxX() - srcBox.getMinX()) / ((double) srcImg.getWidth());
// go through the zoom levels and produce images as needed
for(int x = 0; x < zoomLevels.length; x++){
// get the bounding boxes we will be creating
ArrayList<BoundingBox> boxes = GeoSpatialUtilities.getBoundingBoxSet(zoomLevels[x], srcBox);
// check if the output is correct
if(boxes == null || boxes.size() == 0){
continue;
}
// get the degrees high and wide of an image
double destVertStepDeg = boxes.get(0).getMaxY() - boxes.get(0).getMinY();
double destHorzStepDeg = boxes.get(0).getMaxX() - boxes.get(0).getMinX();
// get the degrees per pixel high and wide
double destVertDPP = destVertStepDeg / ((double) height);
double destHorzDPP = destHorzStepDeg / ((double) width);
// go through each bounding box and produce a partial image
for(int y = 0; y < boxes.size(); y++){
// get the overlap of the two images
BoundingBox overlap = GeoSpatialUtilities.getOverlapBox(boxes.get(y), srcBox);
// check if there is overlap of the images
if(overlap == null){
continue;
}
/*
* determine the offsets for each image
*
* source image image first
*/
// distance in degrees for left side
double srcMinDegX = overlap.getMinX() - srcBox.getMinX();
// minimum x in pixel space
int srcMinPX = (int)(srcMinDegX / srcHorzDPP);
// distance in degrees for right size
double srcMaxDegX = srcBox.getMaxX() - overlap.getMaxX();
// maximum x in pixel space
int srcMaxPX = srcImg.getWidth() - (int)(srcMaxDegX / srcHorzDPP);
// distance in degrees for top
double srcMaxDegY = srcBox.getMaxY() - overlap.getMaxY();
// minimum y pixel
int srcMinPY = (int)(srcMaxDegY / srcVertDPP);
// distance in degrees from bottom
double srcMinDegY = overlap.getMinY() - srcBox.getMinY();
// maximum y pixel
int srcMaxPY = srcImg.getHeight() - (int)(srcMinDegY / srcVertDPP);
/*
* destination image
*/
// distance in degrees for left side
double destMinDegX = overlap.getMinX() - boxes.get(y).getMinX();
// minimum x in pixel space
int destMinPX = (int)(destMinDegX / destHorzDPP);
// distance in degrees for right side
double destMaxDegX = boxes.get(y).getMaxX() - overlap.getMaxX();
// maximum x in pixel space
int destMaxPX = width - (int)(destMaxDegX / destHorzDPP);
// distance in degrees from top
double destMaxDegY = boxes.get(y).getMaxY() - overlap.getMaxY();
// minimum y in pixel space
int destMinPY = (int)(destMaxDegY / destVertDPP);
// distance in degrees from bottom
double destMinDegY = overlap.getMinY() - boxes.get(y).getMinY();
// maximum y in pixel space
int destMaxPY = height - (int)(destMinDegY / destVertDPP);
// create the output image
BufferedImage outImage = new BufferedImage(width, height, BufferedImage.TYPE_4BYTE_ABGR);
// get the graphics to draw the image
Graphics2D outGr = outImage.createGraphics();
// put the source image into the output image
outGr.drawImage(srcImg,
destMinPX, destMinPY, destMaxPX, destMaxPY,
srcMinPX, srcMinPY, srcMaxPX, srcMaxPY,
null);
// serialize the BufferedImage into a byte array
ByteArrayOutputStream baos = new ByteArrayOutputStream();
// write the bytes of the image to the output stream
ImageIO.write(outImage, "png", baos);
// get the bytes of the buffered image
BytesWritable outValue = new BytesWritable(baos.toByteArray());
// create the output key
Text outKey = new Text(GeoSpatialUtilities.getBoundingBoxString(boxes.get(y)) + "_" + width + "x" + height);
// write this portion out
context.write(outKey, outValue);
} // end going through the bounding boxes
} // end zoom levels
} // end map
} // end MapPictures
/**
* ReducePictures is the class that combiles the pieces of image files into the
* final images.
*
* @author alevine@texeltek.com
*
*/
public static class ReducePictures extends org.apache.hadoop.mapreduce.Reducer<Text, BytesWritable, Text, BytesWritable> {
// width of output image
int width = 256;
// height of output image
int height = 256;
/**
* setup() is run before any reducer is run
*/
public void setup(Context context){
// get the configuration of the running system
Configuration conf = context.getConfiguration();
// get the output image width
width = Integer.parseInt(conf.get("imageWidth"));
// get the output image height
height = Integer.parseInt(conf.get("imageHeight"));
} // end setup
/**
* reduce() is the reduce for putting images together
*/
public void reduce(Text key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException{
// create the output image
BufferedImage outImage = null;
// create the output graphics
Graphics2D outGr = null;
// create the output object
BytesWritable outValue = new BytesWritable();
// create the output key
Text outKey = new Text();
// go through the values
for(BytesWritable img : values){
// put images on top of each other
// get the bytes for partial image
byte[] imgBytes = img.getBytes();
// create an input stream to read in bytes
InputStream is = new ByteArrayInputStream(imgBytes);
// get the buffered image from the input bytes
BufferedImage curImg = ImageIO.read(is);
// close the stream
is.close();
// check if the outImage has not been initialized
if(outImage == null){
// initialize the output image
outImage = new BufferedImage(width, height, BufferedImage.TYPE_4BYTE_ABGR);
// initialize the output graphics
outGr = outImage.createGraphics();
}
// add the partial image to the output image
outGr.drawImage(curImg, 0, 0, curImg.getWidth(), curImg.getHeight(), null);
} // end for loop
// serialize the BufferedImage into a byte array
ByteArrayOutputStream baos = new ByteArrayOutputStream();
// write the bytes of the image to the output stream
ImageIO.write(outImage, "png", baos);
// get the bytes of the buffered image and set the output value
outValue.set(new BytesWritable(baos.toByteArray()));
// close output stream
baos.close();
// set the output key
outKey.set(key.toString() + ".png");
// write to the output
context.write(outKey, outValue);
} // end reduce
} // end ReducePictures
/**
* getZoomLevels() will parse the value from the configuration and
* create an array of the integers of the zoom levels to work on
*
* @param str is expected to be in the form 1-2,8,10
* @return an integer array of zoom levels
*/
public static int[] getZoomLevels(String str){
// create a structure to hold the values of zoom levels
ArrayList<Integer> retList = new ArrayList<Integer>();
// split the levels by the comma
String[] els = str.split(",");
// parse each element
for(int x = 0; x < els.length; x++){
// check if the element is a range of zoom levels
if(els[x].contains("-")){
// split the string on the "-"
String[] els2 = els[x].split("-");
// get the start of the zoom levels
int val0 = Integer.parseInt(els2[0]);
// get the end of the zoom levels
int val1 = Integer.parseInt(els2[1]);
// add each value of the zoom level to the output
for(int y = val0; y <= val1; y++){
// check for the minimum and maximum
if(y > 0 && y <= 30){
// add the element
retList.add(new Integer(y));
}
}
} else {
// single value - add it
int val = Integer.parseInt(els[x]);
// check that the value is in the "reasonable" range
if(val > 0 && val <= 30){
// add the element
retList.add(new Integer(val));
}
}
} // end parsing the elements
// get the values as an array
int[] ret = new int[retList.size()];
for(int x = 0; x < retList.size(); x++){
ret[x] = retList.get(x).intValue();
}
return ret;
} // end getZoomLevels
/**
* showUse() will display the correct way to call the program
*
* @param err is the error message
*/
public void showUse(String err){
if(err != null){
// show the error message
System.out.println("Error: " + err);
}
// show the correct way to run the program
System.out.println("Usage: hadoop jar OCCImage.jar org.occ.TileCache.BuildTileCache -i inputDir -o outputDir -zl 1-2,4,8 -r 4");
} // end showUse
/**
* run() is the function that runs the map reduce job
*/
public int run(String[] args) throws Exception {
int numberReducers = 4;
// go through the input arguments
for(int x = 0; x < args.length; x++){
if(args[x].equals("-i")){
// get the input path
inputPath = new Path(args[x+1]);
} else if(args[x].equals("-o")){
// get the output path
outputPath = new Path(args[x+1]);
} else if(args[x].equals("-zl")){
// get the zoom levels to process
zoomLevelsStr = args[x+1];
} else if(args[x].equals("-r")){
// get the number of reducers for the job
numberReducers = Integer.parseInt(args[x+1]);
}
} // end processing input arguments
// check if the input and output directories are set
if(inputPath == null || outputPath == null){
showUse("Problem with directory settings");
return -1;
}
// check if the zoom levels are set
if(zoomLevelsStr == null){
showUse("No zoom levels to work with");
return -1;
}
// get the zoom levels
int[] zoomLevels = getZoomLevels(zoomLevelsStr);
// show the zoom levels that will be processed
System.out.println("Zoom Levels:");
for(int x = 0; x < zoomLevels.length; x++){
System.out.print(" " + zoomLevels[x]);
}
System.out.println();
// get the configuration of the hadoop system
Configuration conf = new Configuration(true);
// set the global values for processing images
conf.set("zoomLevelsStr", zoomLevelsStr);
conf.set("srcprojection", "EPSG:4326");
conf.set("destprojection", "EPSG:4326");
conf.set("imageWidth", "512");
conf.set("imageHeight", "256");
// create the job to be done
Job tileCacheJob = new Job(conf, "Tile Cache for ZL=" + zoomLevelsStr);
// set the input and output format for the job
tileCacheJob.setInputFormatClass(SequenceFileInputFormat.class);
tileCacheJob.setOutputFormatClass(SequenceFileOutputFormat.class);
// set the jar for the job
tileCacheJob.setJarByClass(BuildTileCacheFromSequence.class);
// set the mapper class
tileCacheJob.setMapperClass(BuildTileCacheFromSequence.MapPictures.class);
// set the combiner class - it is possible to combine in this job
tileCacheJob.setCombinerClass(BuildTileCacheFromSequence.ReducePictures.class);
// set the reducer class
tileCacheJob.setReducerClass(BuildTileCacheFromSequence.ReducePictures.class);
// set the output key and value for the mapper
tileCacheJob.setMapOutputKeyClass(Text.class);
tileCacheJob.setMapOutputValueClass(BytesWritable.class);
// set the output key and value for the reducer
tileCacheJob.setOutputKeyClass(Text.class);
tileCacheJob.setOutputValueClass(BytesWritable.class);
// this can be changed to accomodate directing to a reducer
tileCacheJob.setNumReduceTasks(numberReducers);
// set the input path for the job
FileInputFormat.setInputPaths(tileCacheJob, inputPath);
// set the output path for the job
FileOutputFormat.setOutputPath(tileCacheJob, outputPath);
// wait for completion
int retVal = tileCacheJob.waitForCompletion(true) ? 0 : -1;
return retVal;
} // end the run
/**
* main will run the Map-Reduce job
*
* @param args are the command line arguments
*/
public static void main(String[] args){
// run the job
try{
ToolRunner.run(new Configuration(), new BuildTileCacheFromSequence(), args);
} catch(Exception e){
e.printStackTrace();
}
} // end main
} // end main
|