blob: cce86ed3cf99d0e58aea5d78e5551b33d17eaa33 [file] [log] [blame]
/**
* FILE: ImageGenerator.java
* PATH: org.datasyslab.babylon.core.ImageGenerator.java
* Copyright (c) 2015-2017 GeoSpark Development Team
* All rights reserved.
*/
package org.datasyslab.babylon.core;
import java.awt.image.BufferedImage;
import java.io.*;
import java.net.URI;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import org.datasyslab.babylon.utils.ImageType;
import org.datasyslab.babylon.utils.RasterizationUtils;
import org.datasyslab.babylon.utils.S3Operator;
import scala.Tuple2;
import javax.imageio.ImageIO;
// TODO: Auto-generated Javadoc
/**
* The Class ImageGenerator.
*/
public class ImageGenerator implements Serializable{
/** The Constant logger. */
final static Logger logger = Logger.getLogger(ImageGenerator.class);
/**
* Save raster image as local file.
*
* @param distributedImage the distributed image
* @param outputPath the output path
* @param imageType the image type
* @param zoomLevel the zoom level
* @param partitionOnX the partition on X
* @param partitionOnY the partition on Y
* @return true, if successful
* @throws Exception the exception
*/
public boolean SaveRasterImageAsLocalFile(JavaPairRDD<Integer,ImageSerializableWrapper> distributedImage, final String outputPath, final ImageType imageType, final int zoomLevel, final int partitionOnX, final int partitionOnY) throws Exception
{
logger.info("[Babylon][SaveRasterImageAsLocalFile][Start]");
for(int i=0;i<partitionOnX*partitionOnY;i++) {
deleteLocalFile(outputPath+"-"+ RasterizationUtils.getImageTileName(zoomLevel,partitionOnX, partitionOnY,i),imageType);
}
distributedImage.foreach(new VoidFunction<Tuple2<Integer, ImageSerializableWrapper>>() {
@Override
public void call(Tuple2<Integer, ImageSerializableWrapper> integerImageSerializableWrapperTuple2) throws Exception {
SaveRasterImageAsLocalFile(integerImageSerializableWrapperTuple2._2.image, outputPath+"-"+RasterizationUtils.getImageTileName(zoomLevel,partitionOnX, partitionOnY,integerImageSerializableWrapperTuple2._1), imageType);
}
});
logger.info("[Babylon][SaveRasterImageAsLocalFile][Stop]");
return true;
}
/**
* Save raster image as local file.
*
* @param distributedImage the distributed image
* @param outputPath the output path
* @param imageType the image type
* @return true, if successful
* @throws Exception the exception
*/
public boolean SaveRasterImageAsLocalFile(JavaPairRDD<Integer,ImageSerializableWrapper> distributedImage, String outputPath, ImageType imageType) throws Exception
{
List<Tuple2<Integer,ImageSerializableWrapper>> imagePartitions = distributedImage.collect();
for(Tuple2<Integer,ImageSerializableWrapper> imagePartition:imagePartitions)
{
SaveRasterImageAsLocalFile(imagePartition._2.image, outputPath+"-"+imagePartition._1, imageType);
}
return true;
}
/**
* Save raster image as hadoop file.
*
* @param distributedImage the distributed image
* @param outputPath the output path
* @param imageType the image type
* @param zoomLevel the zoom level
* @param partitionOnX the partition on X
* @param partitionOnY the partition on Y
* @return true, if successful
* @throws Exception the exception
*/
public boolean SaveRasterImageAsHadoopFile(JavaPairRDD<Integer,ImageSerializableWrapper> distributedImage, final String outputPath, final ImageType imageType, final int zoomLevel, final int partitionOnX, final int partitionOnY) throws Exception
{
logger.info("[Babylon][SaveRasterImageAsHadoopFile][Start]");
for(int i=0;i<partitionOnX*partitionOnY;i++) {
deleteHadoopFile(outputPath+"-"+RasterizationUtils.getImageTileName(zoomLevel,partitionOnX, partitionOnY,i)+".", imageType);
}
distributedImage.foreach(new VoidFunction<Tuple2<Integer, ImageSerializableWrapper>>() {
@Override
public void call(Tuple2<Integer, ImageSerializableWrapper> integerImageSerializableWrapperTuple2) throws Exception {
SaveRasterImageAsHadoopFile(integerImageSerializableWrapperTuple2._2.image, outputPath+"-"+RasterizationUtils.getImageTileName(zoomLevel,partitionOnX, partitionOnY,integerImageSerializableWrapperTuple2._1), imageType);
}
});
logger.info("[Babylon][SaveRasterImageAsHadoopFile][Stop]");
return true;
}
/**
* Save raster image as S 3 file.
*
* @param distributedImage the distributed image
* @param regionName the region name
* @param accessKey the access key
* @param secretKey the secret key
* @param bucketName the bucket name
* @param path the path
* @param imageType the image type
* @param zoomLevel the zoom level
* @param partitionOnX the partition on X
* @param partitionOnY the partition on Y
* @return true, if successful
*/
public boolean SaveRasterImageAsS3File(JavaPairRDD<Integer,ImageSerializableWrapper> distributedImage,
final String regionName, final String accessKey, final String secretKey,
final String bucketName, final String path, final ImageType imageType, final int zoomLevel, final int partitionOnX, final int partitionOnY)
{
logger.info("[Babylon][SaveRasterImageAsS3File][Start]");
S3Operator s3Operator = new S3Operator(regionName, accessKey, secretKey);
for(int i=0;i<partitionOnX*partitionOnY;i++) {
s3Operator.deleteImage(bucketName, path+"-"+RasterizationUtils.getImageTileName(zoomLevel,partitionOnX, partitionOnY,i)+"."+imageType.getTypeName());
}
distributedImage.foreach(new VoidFunction<Tuple2<Integer, ImageSerializableWrapper>>() {
@Override
public void call(Tuple2<Integer, ImageSerializableWrapper> integerImageSerializableWrapperTuple2) throws Exception {
SaveRasterImageAsS3File(integerImageSerializableWrapperTuple2._2.image, regionName, accessKey, secretKey, bucketName, path+"-"+RasterizationUtils.getImageTileName(zoomLevel,partitionOnX, partitionOnY,integerImageSerializableWrapperTuple2._1), imageType);
}
});
logger.info("[Babylon][SaveRasterImageAsS3File][Stop]");
return true;
}
/**
* Save raster image as local file.
*
* @param rasterImage the raster image
* @param outputPath the output path
* @param imageType the image type
* @return true, if successful
* @throws Exception the exception
*/
public boolean SaveRasterImageAsLocalFile(BufferedImage rasterImage, String outputPath, ImageType imageType) throws Exception
{
logger.info("[Babylon][SaveRasterImageAsLocalFile][Start]");
File outputImage = new File(outputPath+"."+imageType.getTypeName());
outputImage.getParentFile().mkdirs();
try {
ImageIO.write(rasterImage,imageType.getTypeName(),outputImage);
} catch (IOException e) {
e.printStackTrace();
}
logger.info("[Babylon][SaveRasterImageAsLocalFile][Stop]");
return true;
}
/**
* Save raster image as hadoop file.
*
* @param rasterImage the raster image
* @param originalOutputPath the original output path
* @param imageType the image type
* @return true, if successful
* @throws Exception the exception
*/
public boolean SaveRasterImageAsHadoopFile(BufferedImage rasterImage, String originalOutputPath, ImageType imageType) throws Exception
{
logger.info("[Babylon][SaveRasterImageAsHadoopFile][Start]");
// Locate HDFS path
String outputPath = originalOutputPath+"."+imageType.getTypeName();
String[] splitString = outputPath.split(":");
String hostName = splitString[0]+":"+splitString[1];
String[] portAndPath = splitString[2].split("/");
String port = portAndPath[0];
String localPath = "";
for(int i=1;i<portAndPath.length;i++)
{
localPath+="/"+portAndPath[i];
}
localPath+="."+imageType.getTypeName();
// Delete existing files
Configuration hadoopConf = new org.apache.hadoop.conf.Configuration();
logger.info("[Babylon][SaveRasterImageAsSparkFile] HDFS URI BASE: "+hostName+":"+port);
FileSystem hdfs = FileSystem.get(new URI(hostName+":"+port), hadoopConf);
logger.info("[Babylon][SaveRasterImageAsSparkFile] Check the existence of path: "+localPath);
if (hdfs.exists(new org.apache.hadoop.fs.Path(localPath))) {
logger.info("[Babylon][SaveRasterImageAsSparkFile] Deleting path: "+localPath);
hdfs.delete(new org.apache.hadoop.fs.Path(localPath), true);
logger.info("[Babylon][SaveRasterImageAsSparkFile] Deleted path: "+localPath);
}
Path path = new Path(outputPath);
FSDataOutputStream out = hdfs.create(path);
ImageIO.write(rasterImage,"png",out);
out.close();
hdfs.close();
logger.info("[Babylon][SaveRasterImageAsHadoopFile][Stop]");
return true;
}
/**
* Save raster image as S 3 file.
*
* @param rasterImage the raster image
* @param regionName the region name
* @param accessKey the access key
* @param secretKey the secret key
* @param bucketName the bucket name
* @param path the path
* @param imageType the image type
* @return true, if successful
* @throws IOException Signals that an I/O exception has occurred.
*/
public boolean SaveRasterImageAsS3File(BufferedImage rasterImage, String regionName, String accessKey, String secretKey, String bucketName, String path, ImageType imageType) throws IOException {
logger.info("[Babylon][SaveRasterImageAsS3File][Start]");
S3Operator s3Operator = new S3Operator(regionName,accessKey, secretKey);
s3Operator.putImage(bucketName,path+"."+imageType.getTypeName(),rasterImage);
logger.info("[Babylon][SaveRasterImageAsS3File][Stop]");
return true;
}
/**
* Save vector image as local file.
*
* @param distributedImage the distributed image
* @param outputPath the output path
* @param imageType the image type
* @return true, if successful
* @throws Exception the exception
*/
public boolean SaveVectorImageAsLocalFile(JavaPairRDD<Integer,String> distributedImage, String outputPath, ImageType imageType) throws Exception
{
logger.info("[Babylon][SaveVectormageAsLocalFile][Start]");
JavaRDD<String> distributedVectorImageNoKey= distributedImage.map(new Function<Tuple2<Integer,String>, String>()
{
@Override
public String call(Tuple2<Integer, String> vectorObject) throws Exception {
return vectorObject._2();
}
});
this.SaveVectorImageAsLocalFile(distributedVectorImageNoKey.collect(), outputPath, imageType);
logger.info("[Babylon][SaveVectormageAsLocalFile][Stop]");
return true;
}
/**
* Save vector image as local file.
*
* @param vectorImage the vector image
* @param outputPath the output path
* @param imageType the image type
* @return true, if successful
* @throws Exception the exception
*/
public boolean SaveVectorImageAsLocalFile(List<String> vectorImage, String outputPath, ImageType imageType) throws Exception
{
logger.info("[Babylon][SaveVectorImageAsLocalFile][Start]");
File outputImage = new File(outputPath+"."+imageType.getTypeName());
outputImage.getParentFile().mkdirs();
BufferedWriter bw = null;
FileWriter fw = null;
try {
fw = new FileWriter(outputImage);
bw = new BufferedWriter(fw);
for(String svgElement : vectorImage)
{
bw.write(svgElement);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (bw != null)
bw.close();
if (fw != null)
fw.close();
} catch (IOException ex) {
ex.printStackTrace();
}
}
logger.info("[Babylon][SaveVectorImageAsLocalFile][Stop]");
return true;
}
/**
* Delete hadoop file.
*
* @param originalOutputPath the original output path
* @param imageType the image type
* @return true, if successful
* @throws Exception the exception
*/
public boolean deleteHadoopFile(String originalOutputPath, ImageType imageType) throws Exception {
String outputPath = originalOutputPath+"."+imageType.getTypeName();
String[] splitString = outputPath.split(":");
String hostName = splitString[0]+":"+splitString[1];
String[] portAndPath = splitString[2].split("/");
String port = portAndPath[0];
String localPath = "";
for(int i=1;i<portAndPath.length;i++)
{
localPath+="/"+portAndPath[i];
}
localPath+="."+imageType.getTypeName();
// Delete existing files
Configuration hadoopConf = new org.apache.hadoop.conf.Configuration();
logger.info("[Babylon][SaveRasterImageAsSparkFile] HDFS URI BASE: "+hostName+":"+port);
FileSystem hdfs = FileSystem.get(new URI(hostName+":"+port), hadoopConf);
logger.info("[Babylon][SaveRasterImageAsSparkFile] Check the existence of path: "+localPath);
if (hdfs.exists(new org.apache.hadoop.fs.Path(localPath))) {
logger.info("[Babylon][SaveRasterImageAsSparkFile] Deleting path: "+localPath);
hdfs.delete(new org.apache.hadoop.fs.Path(localPath), true);
logger.info("[Babylon][SaveRasterImageAsSparkFile] Deleted path: "+localPath);
}
return true;
}
/**
* Delete local file.
*
* @param originalOutputPath the original output path
* @param imageType the image type
* @return true, if successful
*/
public boolean deleteLocalFile(String originalOutputPath, ImageType imageType) {
File file = null;
try {
// create new file
file = new File(originalOutputPath+"."+imageType.getTypeName());
// tries to delete a non-existing file
file.delete();
} catch(Exception e) {
// if any error occurs
e.printStackTrace();
}
return true;
}
}