List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext getJobName
public String getJobName();
From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java
License:Apache License
private void doAbortTask(TaskAttemptContext context, OutputCommitter committer) { try {/* ww w. j a v a 2 s. c o m*/ committer.abortTask(context); } catch (IOException e) { LOG.error(MessageFormat.format("error occurred while aborting task: {0} ({1})", context.getTaskAttemptID(), context.getJobName()), e); } }
From source file:com.bigdata.mapreduce.seqtotext.beta.ZipFileRecordReader.java
License:Apache License
/** * Initialise and open the ZIP file from the FileSystem *//*from w ww . j av a2 s . c o m*/ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { // Configuration conf = new Configuration(); // conf.set("fs.defaultFS", "hdfs://training.musigma.com:8020/user/musigma/"); FileSplit split = (FileSplit) inputSplit; System.out.println("the task attempt instance is : " + taskAttemptContext.getJobName()); System.out.println("the task attempt instance is : " + taskAttemptContext.getWorkingDirectory().toString()); Configuration conf = taskAttemptContext.getConfiguration(); Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); System.out.println("file system replication : " + fs.getDefaultReplication()); // Open the stream fsin = fs.open(path); // zip = new ZipInputStream(fsin); tar = new TarInputStream(fsin); System.out.println("tar input stream is : " + tar.toString()); }
From source file:com.datasalt.pangool.solr.SolrRecordWriter.java
License:Apache License
public SolrRecordWriter(int batchSize, boolean outputZipFile, int threadCount, int queueSize, String localSolrHome, String zipName, TupleDocumentConverter converter, TaskAttemptContext context) { this.localSolrHome = localSolrHome; this.zipName = zipName; conf = context.getConfiguration();/* w w w .j a va2 s . c o m*/ this.batchSize = batchSize; setLogLevel("org.apache.solr.core", "WARN"); setLogLevel("org.apache.solr.update", "WARN"); Logger.getLogger("org.apache.solr.core").setLevel(Level.WARN); Logger.getLogger("org.apache.solr.update").setLevel(Level.WARN); java.util.logging.Logger.getLogger("org.apache.solr.core").setLevel(java.util.logging.Level.WARNING); java.util.logging.Logger.getLogger("org.apache.solr.update").setLevel(java.util.logging.Level.WARNING); setLogLevel("org.apache.solr", "WARN"); Logger.getLogger("org.apache.solr").setLevel(Level.WARN); java.util.logging.Logger.getLogger("org.apache.solr").setLevel(java.util.logging.Level.WARNING); heartBeater = new HeartBeater(context); try { heartBeater.needHeartBeat(); /** The actual file in hdfs that holds the configuration. */ this.outputZipFile = outputZipFile; this.fs = FileSystem.get(conf); perm = new Path(FileOutputFormat.getOutputPath(context), getOutFileName(context, "part")); // Make a task unique name that contains the actual index output name to // make debugging simpler // Note: if using JVM reuse, the sequence number will not be reset for a // new task using the jvm Path temp = conf.getLocalPath("mapred.local.dir", "solr_" + conf.get("mapred.task.id") + '.' + sequence.incrementAndGet()); if (outputZipFile && !perm.getName().endsWith(".zip")) { perm = perm.suffix(".zip"); } fs.delete(temp, true); // delete old, if any fs.delete(perm, true); // delete old, if any local = fs.startLocalOutput(perm, temp); solrHome = findSolrConfig(conf); // } // Verify that the solr home has a conf and lib directory if (solrHome == null) { throw new IOException("Unable to find solr home setting"); } // Setup a solr instance that we can batch writes to LOG.info("SolrHome: " + solrHome.toUri()); String dataDir = new File(local.toString(), "data").getAbsoluteFile().toString(); // copy the schema to the conf dir File confDir = new File(local.toString(), "conf"); confDir.mkdirs(); File unpackedSolrHome = new File(solrHome.toString()); FileUtils.copyDirectory(new File(unpackedSolrHome, "conf"), confDir); Properties props = new Properties(); props.setProperty("solr.data.dir", dataDir); props.setProperty("solr.home", solrHome.toString()); SolrResourceLoader loader = new SolrResourceLoader(solrHome.toString(), null, props); LOG.info(String.format( "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to temporary directory %s, with permdir %s", solrHome, solrHome.toUri(), loader.getInstanceDir(), loader.getConfigDir(), dataDir, perm)); CoreContainer container = new CoreContainer(loader); CoreDescriptor descr = new CoreDescriptor(container, "core1", solrHome.toString()); descr.setDataDir(dataDir); descr.setCoreProperties(props); core = container.create(descr); container.register(core, false); solr = new EmbeddedSolrServer(container, "core1"); batchWriter = new BatchWriter(solr, batchSize, context.getTaskAttemptID().getTaskID(), threadCount, queueSize); this.converter = converter; } catch (Exception e) { e.printStackTrace(); LOG.error(e); throw new IllegalStateException(String.format("Failed to initialize record writer for %s, %s", context.getJobName(), conf.get("mapred.task.id")), e); } finally { heartBeater.cancelHeartBeat(); } }
From source file:com.vertica.hadoop.VerticaOutputFormat.java
License:Apache License
/** {@inheritDoc} */ public RecordWriter<Text, VerticaRecord> getRecordWriter(TaskAttemptContext context) throws IOException { VerticaConfiguration config = new VerticaConfiguration(context.getConfiguration()); String name = context.getJobName(); String table = config.getOutputTableName(); try {/*from w w w . ja v a2 s .c o m*/ return new VerticaRecordWriter(getConnection(context.getConfiguration()), table, config.getBatchSize()); } catch (SQLException e) { throw new IOException(e); } }
From source file:org.apache.solr.hadoop.SolrRecordWriter.java
License:Apache License
public SolrRecordWriter(TaskAttemptContext context, Path outputShardDir, int batchSize) { this.batchSize = batchSize; this.batch = new ArrayList(batchSize); Configuration conf = context.getConfiguration(); // setLogLevel("org.apache.solr.core", "WARN"); // setLogLevel("org.apache.solr.update", "WARN"); heartBeater = new HeartBeater(context); try {// w ww . jav a 2s. c o m heartBeater.needHeartBeat(); Path solrHomeDir = SolrRecordWriter.findSolrConfig(conf); FileSystem fs = outputShardDir.getFileSystem(conf); EmbeddedSolrServer solr = createEmbeddedSolrServer(solrHomeDir, fs, outputShardDir); batchWriter = new BatchWriter(solr, batchSize, context.getTaskAttemptID().getTaskID(), SolrOutputFormat.getSolrWriterThreadCount(conf), SolrOutputFormat.getSolrWriterQueueSize(conf)); } catch (Exception e) { throw new IllegalStateException(String.format("Failed to initialize record writer for %s, %s", context.getJobName(), conf.get("mapred.task.id")), e); } finally { heartBeater.cancelHeartBeat(); } }
From source file:org.gbif.ocurrence.index.solr.SolrRecordWriter.java
License:Apache License
@SuppressWarnings("unchecked") public SolrRecordWriter(TaskAttemptContext context) { conf = context.getConfiguration();/*from w w w.j a va 2 s .c o m*/ batchSize = SolrOutputFormat.getBatchSize(conf); setLogLevel("org.apache.solr.core", "WARN"); setLogLevel("org.apache.solr.update", "WARN"); heartBeater = new HeartBeater(context); try { heartBeater.needHeartBeat(); /** The actual file in hdfs that holds the configuration. */ final String configuredSolrConfigPath = conf.get(SolrOutputFormat.SETUP_OK); if (configuredSolrConfigPath == null) { throw new IllegalStateException( String.format("The job did not pass %s", SolrOutputFormat.SETUP_OK)); } outputZipFile = SolrOutputFormat.isOutputZipFormat(conf); this.fs = FileSystem.get(conf); perm = new Path(FileOutputFormat.getOutputPath(context), getOutFileName(context, "part")); // Make a task unique name that contains the actual index output name to // make debugging simpler // Note: if using JVM reuse, the sequence number will not be reset for a // new task using the jvm temp = conf.getLocalPath("mapred.local.dir", "solr_" + conf.get("mapred.task.id") + '.' + sequence.incrementAndGet()); if (outputZipFile && !perm.getName().endsWith(".zip")) { perm = perm.suffix(".zip"); } fs.delete(perm, true); // delete old, if any Path local = fs.startLocalOutput(perm, temp); solrHome = findSolrConfig(conf); // } // Verify that the solr home has a conf and lib directory if (solrHome == null) { throw new IOException("Unable to find solr home setting"); } // Setup a solr instance that we can batch writes to LOG.info("SolrHome: " + solrHome.toUri()); String dataDir = new File(local.toString(), "data").toString(); // copy the schema to the conf dir File confDir = new File(local.toString(), "conf"); confDir.mkdirs(); File srcSchemaFile = new File(solrHome.toString(), "conf/schema.xml"); assert srcSchemaFile.exists(); FileUtils.copyFile(srcSchemaFile, new File(confDir, "schema.xml")); Properties props = new Properties(); props.setProperty("solr.data.dir", dataDir); props.setProperty("solr.home", solrHome.toString()); SolrResourceLoader loader = new SolrResourceLoader(solrHome.toString(), null, props); LOG.info(String.format( "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to temporary directory %s, with permdir %s", solrHome, solrHome.toUri(), loader.getInstanceDir(), loader.getConfigDir(), dataDir, perm)); CoreContainer container = new CoreContainer(loader); CoreDescriptor descr = new CoreDescriptor(container, "core1", solrHome.toString()); descr.setDataDir(dataDir); descr.setCoreProperties(props); core = container.create(descr); container.register(core, false); solr = new EmbeddedSolrServer(container, "core1"); batchWriter = new BatchWriter(solr, batchSize, context.getTaskAttemptID().getTaskID(), SolrOutputFormat.getSolrWriterThreadCount(conf), SolrOutputFormat.getSolrWriterQueueSize(conf)); // instantiate the converter String className = SolrDocumentConverter.getSolrDocumentConverter(conf); Class<? extends SolrDocumentConverter> cls = (Class<? extends SolrDocumentConverter>) Class .forName(className); converter = (SolrDocumentConverter<K, V>) ReflectionUtils.newInstance(cls, conf); } catch (Exception e) { throw new IllegalStateException(String.format("Failed to initialize record writer for %s, %s", context.getJobName(), conf.get("mapred.task.id")), e); } finally { heartBeater.cancelHeartBeat(); } }