Java tutorial
/* * Druid - a distributed column store. * Copyright (C) 2012 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package com.metamx.druid.indexer; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import com.metamx.common.ISE; import com.metamx.druid.jackson.DefaultObjectMapper; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.ReflectionUtils; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.HashMap; import java.util.Iterator; import java.util.Map; /** */ public class Utils { private static final ObjectMapper jsonMapper = new DefaultObjectMapper(); public static <K, V> Map<K, V> zipMap(Iterable<K> keys, Iterable<V> values) { Map<K, V> retVal = new HashMap<K, V>(); Iterator<K> keyIter = keys.iterator(); Iterator<V> valsIter = values.iterator(); while (keyIter.hasNext()) { final K key = keyIter.next(); Preconditions.checkArgument(valsIter.hasNext(), "keys longer than vals, bad, bad vals. Broke on key[%s]", key); retVal.put(key, valsIter.next()); } if (valsIter.hasNext()) { throw new ISE("More values[%d] than keys[%d]", retVal.size() + Iterators.size(valsIter), retVal.size()); } return retVal; } public static OutputStream makePathAndOutputStream(JobContext job, Path outputPath, boolean deleteExisting) throws IOException { OutputStream retVal; FileSystem fs = outputPath.getFileSystem(job.getConfiguration()); if (fs.exists(outputPath)) { if (deleteExisting) { fs.delete(outputPath, false); } else { throw new ISE("outputPath[%s] must not exist.", outputPath); } } if (!FileOutputFormat.getCompressOutput(job)) { retVal = fs.create(outputPath, false); } else { Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(job, GzipCodec.class); CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job.getConfiguration()); outputPath = new Path(outputPath.toString() + codec.getDefaultExtension()); retVal = codec.createOutputStream(fs.create(outputPath, false)); } return retVal; } public static InputStream openInputStream(JobContext job, Path inputPath) throws IOException { return openInputStream(inputPath, inputPath.getFileSystem(job.getConfiguration())); } public static InputStream openInputStream(Path inputPath, final FileSystem fileSystem) throws IOException { return fileSystem.open(inputPath); } public static Map<String, Object> getStats(JobContext job, Path statsPath) throws IOException { FileSystem fs = statsPath.getFileSystem(job.getConfiguration()); return jsonMapper.readValue(fs.open(statsPath), new TypeReference<Map<String, Object>>() { }); } public static void storeStats(JobContext job, Path path, Map<String, Object> stats) throws IOException { jsonMapper.writeValue(makePathAndOutputStream(job, path, true), stats); } }