List of usage examples for org.apache.hadoop.fs FileSystem getLocal
public static LocalFileSystem getLocal(Configuration conf) throws IOException
From source file:com.moz.fiji.schema.tools.CreateTableTool.java
License:Apache License
/** {@inheritDoc} */ @Override//from w w w. j a v a2 s . co m protected int run(List<String> nonFlagArgs) throws Exception { getPrintStream().println("Parsing table layout: " + mLayout); final Path path = new Path(mLayout); final FileSystem fs = fileSystemSpecified(path) ? path.getFileSystem(getConf()) : FileSystem.getLocal(getConf()); final FSDataInputStream inputStream = fs.open(path); final TableLayoutDesc tableLayout = FijiTableLayout.readTableLayoutDescFromJSON(inputStream); final String tableName = tableLayout.getName(); Preconditions.checkArgument((mTableURI.getTable() == null) || tableName.equals(mTableURI.getTable()), "Table name '%s' does not match URI %s", tableName, mTableURI); // For large numbers of initial regions, table creation may take a long time as we wait for // the new regions to come online. Increase the hbase RPC timeout to compensate. int hbaseTimeout = getConf().getInt("hbase.rpc.timeout", 60000); hbaseTimeout = hbaseTimeout * 10; getConf().setInt("hbase.rpc.timeout", hbaseTimeout); getPrintStream().println("Creating Fiji table " + mTableURI); if (mNumRegions >= 1) { // Create a table with an initial number of evenly split regions. mFiji.createTable(tableLayout, mNumRegions); } else if (!mSplitKeyFilePath.isEmpty()) { switch (FijiTableLayout.getEncoding(tableLayout.getKeysFormat())) { case HASH: case HASH_PREFIX: throw new IllegalArgumentException( "Row key hashing is enabled for the table. Use --num-regions=N instead."); case RAW: break; case FORMATTED: // TODO Support pre-splitting tables for FORMATTED RKF // (https://jira.fiji.org/browse/SCHEMA-172) throw new RuntimeException("CLI support for FORMATTED row keys is not yet available"); default: throw new RuntimeException( "Unexpected row key encoding: " + FijiTableLayout.getEncoding(tableLayout.getKeysFormat())); } // Open the split key file. final Path splitKeyFilePath = new Path(mSplitKeyFilePath); final FileSystem splitKeyPathFs = fileSystemSpecified(splitKeyFilePath) ? splitKeyFilePath.getFileSystem(getConf()) : FileSystem.getLocal(getConf()); final FSDataInputStream splitKeyFileInputStream = splitKeyPathFs.open(splitKeyFilePath); // Read the split keys. final List<byte[]> splitKeys = SplitKeyFile.decodeRegionSplitList(splitKeyFileInputStream); LOG.debug("Read {} keys from split-key-file '{}':", splitKeys.size(), splitKeyFilePath); for (int i = 0; i < splitKeys.size(); ++i) { LOG.debug("Split key #{}: {}", i, Bytes.toStringBinary(splitKeys.get(i))); } // Create the table with the given split keys. mFiji.createTable(tableLayout, splitKeys.toArray(new byte[splitKeys.size()][])); } else { // Create a table with a single initial region: mFiji.createTable(tableLayout); } return SUCCESS; }
From source file:com.moz.fiji.schema.tools.LayoutTool.java
License:Apache License
/** * Loads a table layout descriptor from a JSON-encoded file. * * @param filePath Path to a JSON-encoded table layout descriptor. * @return the table layout descriptor decoded from the file. * @throws Exception on error.//from www . j a va2 s . c om */ private TableLayoutDesc loadJsonTableLayoutDesc(String filePath) throws Exception { final Path path = new Path(filePath); final FileSystem fs = fileSystemSpecified(path) ? path.getFileSystem(getConf()) : FileSystem.getLocal(getConf()); final InputStream istream = fs.open(path); try { return FijiTableLayout.readTableLayoutDescFromJSON(istream); } finally { ResourceUtils.closeOrLog(istream); ResourceUtils.closeOrLog(fs); } }
From source file:com.mycompany.mavenpails2.PailMove.java
public static void main(String args[]) throws Exception { //Los primeros pasos son establecer la configuracin de Hadoop. setApplicationConf();// w w w . j a va2s . c om LocalFileSystem fs = FileSystem.getLocal(new Configuration()); /* Luego creamos Los dos pails necesarios para alojar el masterdataset NewDataPail contendr los registros nuevos y stos se aadirn al MasterPail, que es el master dataset. */ Pail newDataPail; Pail masterPail; Path fils = new Path(NEW_DATA_LOCATION); if (!fs.exists(fils)) { newDataPail = Pail.create(FileSystem.get(new Configuration()), NEW_DATA_LOCATION, new DataPailStructure()); } else { newDataPail = new Pail<Data>(NEW_DATA_LOCATION); } if (!fs.exists(new Path(MASTER_DATA_LOCATION))) { masterPail = Pail.create(FileSystem.getLocal(new Configuration()), MASTER_DATA_LOCATION, new SplitDataPailStructure()); } else { masterPail = new Pail<Data>(MASTER_DATA_LOCATION); } /* La siguiente rutina toma un archivo de texto de la carpeta resources y la desmembra para insertar sus datos en el newDataPail. */ Pail.TypedRecordOutputStream out = newDataPail.openWrite(); PailMove c = new PailMove(); Class cls = c.getClass(); File file = new File(cls.getClassLoader().getResource("realdataset.txt").getFile()); try (Scanner scanner = new Scanner(file)) { while (scanner.hasNextLine()) { String line = scanner.nextLine(); //result.append(line).append("\n"); StringTokenizer tkn = new StringTokenizer(line); String sid = tkn.nextToken(); String stime = tkn.nextToken(); String stipo = tkn.nextToken(); String seje1 = tkn.nextToken(); String seje2 = tkn.nextToken(); String selev = tkn.nextToken(); String sx = tkn.nextToken(); String sy = tkn.nextToken(); String sz = tkn.nextToken(); String svalue = tkn.nextToken(); if (!tkn.hasMoreTokens()) { long id = Long.parseLong(sid); int time = Integer.parseInt(stime); int tipo = Integer.parseInt(stipo); int eje1 = Integer.parseInt(seje1); int eje2 = Integer.parseInt(seje2); int elev = Integer.parseInt(selev); int posx = Integer.parseInt(sx); int posy = Integer.parseInt(sy); int posz = Integer.parseInt(sz); double value = Double.parseDouble(svalue); out.writeObject(GenerateData.setValue(id, time, value)); out.writeObject(GenerateData.setTipo(id, time, tipo)); out.writeObject(GenerateData.setPos(id, time, eje1, eje2, elev, posx, posy, posz)); } } out.close(); scanner.close(); } catch (IOException e) { e.printStackTrace(); } // shred(); /* Ingest es una serie de pasos para finalmente insertar el newData en el master dataset, evitando duplicancia de datos y corrupcin. */ ingest(masterPail, newDataPail); // Prepwork crea un join de datos con el fin de cclular las Batch Views prepWork(); //Finalmente se calculan las Batch Views. //Object source = Api.hfsSeqfile("/tmp/accel"); Api.execute(new StdoutTap(), TempBatchView()); Api.execute(new StdoutTap(), AnemBatchView1()); Api.execute(new StdoutTap(), AnemBatchView2()); Api.execute(new StdoutTap(), AccelBatchView1()); accelElephantDB(AccelBatchView1()); /* Elminiamos el directorio temporal joins (Debe haber una mejor forma de hacer esto) */ File index = new File("/tmp/joins"); deleteFolder(index); /* CassandraDatastax client = new CassandraDatastax(); client.connect("127.0.0.1"); client.loadData(); client.close(); */ }
From source file:com.myhp.hive.rcfile.RCFileGenerator.java
License:Apache License
public static void main(String[] args) throws Exception { if (args.length != 4) usage();// w ww . j ava 2s . c o m String format = args[0]; int numRows = Integer.valueOf(args[1]); if (numRows < 1) usage(); String output = args[2]; String plainOutput = args[3]; fs = FileSystem.getLocal(conf); basedir = new Path("."); genData(format, numRows, output, plainOutput); }
From source file:com.netflix.bdp.s3.MockedS3Committer.java
License:Apache License
@Override public void commitJob(JobContext context) throws IOException { super.commitJob(context); Configuration conf = context.getConfiguration(); try {//from w w w . ja va 2 s . c o m String jobCommitterPath = conf.get("mock-results-file"); if (jobCommitterPath != null) { try (ObjectOutputStream out = new ObjectOutputStream( FileSystem.getLocal(conf).create(new Path(jobCommitterPath), false))) { out.writeObject(results); } } } catch (Exception e) { // do nothing, the test will fail } }
From source file:com.netflix.bdp.s3.TestMRJob.java
License:Apache License
@Test public void testMRJob() throws Exception { FileSystem mockS3 = mock(FileSystem.class); FileSystem s3 = S3_OUTPUT_PATH.getFileSystem(getConfiguration()); if (s3 instanceof MockS3FileSystem) { ((MockS3FileSystem) s3).setMock(mockS3); } else {/*from w ww .j a v a2s . c om*/ throw new RuntimeException("Cannot continue: S3 not mocked"); } String commitUUID = UUID.randomUUID().toString(); int numFiles = 3; Set<String> expectedFiles = Sets.newHashSet(); for (int i = 0; i < numFiles; i += 1) { File file = temp.newFile(String.valueOf(i) + ".text"); try (FileOutputStream out = new FileOutputStream(file)) { out.write(("file " + i).getBytes(StandardCharsets.UTF_8)); } expectedFiles.add(new Path(S3_OUTPUT_PATH, "part-m-0000" + i + "-" + commitUUID).toString()); } Job mrJob = Job.getInstance(MR_CLUSTER.getConfig(), "test-committer-job"); Configuration conf = mrJob.getConfiguration(); mrJob.setOutputFormatClass(S3TextOutputFormat.class); S3TextOutputFormat.setOutputPath(mrJob, S3_OUTPUT_PATH); File mockResultsFile = temp.newFile("committer.bin"); mockResultsFile.delete(); String committerPath = "file:" + mockResultsFile; conf.set("mock-results-file", committerPath); conf.set(UPLOAD_UUID, commitUUID); mrJob.setInputFormatClass(TextInputFormat.class); TextInputFormat.addInputPath(mrJob, new Path("file:" + temp.getRoot().toString())); mrJob.setMapperClass(M.class); mrJob.setNumReduceTasks(0); mrJob.submit(); Assert.assertTrue("MR job should succeed", mrJob.waitForCompletion(true)); TestUtil.ClientResults results; try (ObjectInputStream in = new ObjectInputStream( FileSystem.getLocal(conf).open(new Path(committerPath)))) { results = (TestUtil.ClientResults) in.readObject(); } Assert.assertEquals("Should not delete files", 0, results.deletes.size()); Assert.assertEquals("Should not abort commits", 0, results.aborts.size()); Assert.assertEquals("Should commit task output files", numFiles, results.commits.size()); Set<String> actualFiles = Sets.newHashSet(); for (CompleteMultipartUploadRequest commit : results.commits) { actualFiles.add("s3://" + commit.getBucketName() + "/" + commit.getKey()); } Assert.assertEquals("Should commit the correct file paths", expectedFiles, actualFiles); }
From source file:com.netflix.suro.sink.localfile.FileWriterBase.java
License:Apache License
public FileWriterBase(String codecClass, Logger log, Configuration conf) { this.conf = conf; try {/*from w ww. j a v a2 s . c o m*/ fs = FileSystem.getLocal(conf); fs.setVerifyChecksum(false); if (codecClass != null) { codec = createCodecInstance(codecClass); log.info("Codec:" + codec.getDefaultExtension()); } else { codec = null; } } catch (Exception e) { throw new RuntimeException(e); } }
From source file:com.ngdata.sep.impl.SepConsumerIT.java
License:Apache License
static String getDefaultUmask() throws IOException { // Hack to get around the test DFS cluster only wanting to start up if the // umask is set to the expected value (i.e. 022) File tmpDir = Files.createTempDir(); LocalFileSystem local = FileSystem.getLocal(new Configuration()); FileStatus stat = local.getFileStatus(new Path(tmpDir.getAbsolutePath())); FsPermission permission = stat.getPermission(); String permString = Integer.toString(permission.toShort(), 8); tmpDir.delete();/* w w w.j a va2 s. c om*/ return permString; }
From source file:com.pegasus.ResultInfo.java
License:Apache License
public int run(final String[] args) throws Exception { Configuration conf = getConf(); final FileSystem fs = FileSystem.get(conf); edge_path = new Path(conf.get("edge_path")); all_vertices = new Path(conf.get("all_vertices")); curbm_path = new Path(conf.get("iteration_state")); tempbm_path = new Path(conf.get("stage1out")); nextbm_path = new Path(conf.get("stage2out")); output_path = new Path(conf.get("stage3out")); grapherOut_path = new Path(conf.get("grapherout")); nreducers = Integer.parseInt(conf.get("num_reducers")); local_output_path = conf.get("local_output"); // initital cleanup fs.delete(tempbm_path, true);/*from w w w .j a v a2s . co m*/ fs.delete(nextbm_path, true); fs.delete(output_path, true); fs.delete(curbm_path, true); fs.delete(grapherOut_path, true); FileUtil.fullyDelete(new File(local_output_path)); fs.mkdirs(curbm_path); //fs.mkdirs(grapherOut_path); FileStatus[] statusArray = fs.listStatus(all_vertices); for (int index = 0; index < statusArray.length; index++) { Path temp = statusArray[index].getPath(); FileUtil.copy(fs, temp, fs, curbm_path, false, conf); } make_symmetric = 1; System.out.println("\n-----===[PEGASUS: A Peta-Scale Graph Mining System]===-----\n"); // Iteratively calculate neighborhood function. // rotate directory for (int i = cur_iter; i < MAX_ITERATIONS; i++) { cur_iter++; System.out.println("configStage1"); JobClient.runJob(configStage1()); System.out.println("configStage2"); JobClient.runJob(configStage2()); System.out.println("configStage3"); JobClient.runJob(configStage3()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); // copy neighborhood information from HDFS to local disk, and read it! String new_path = local_output_path + "/" + i; fs.copyToLocalFile(output_path, new Path(new_path)); ResultInfo ri = readIterationOutput(new_path); changed_nodes[iter_counter] = ri.changed; changed_nodes[iter_counter] = ri.unchanged; iter_counter++; System.out.println("Hop " + i + " : changed = " + ri.changed + ", unchanged = " + ri.unchanged); fs.delete(curbm_path); fs.delete(tempbm_path); fs.delete(output_path); fs.rename(nextbm_path, curbm_path); // Stop when the minimum neighborhood doesn't change if (ri.changed == 0) { System.out.println("All the component ids converged. Finishing..."); fs.rename(curbm_path, grapherOut_path); break; } } FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); // finishing. System.out.println("\n[PEGASUS] Connected component computed."); System.out.println("[PEGASUS] Total Iteration = " + iter_counter); return 0; }
From source file:com.pivotal.hawq.mapreduce.MapReduceLocalDriver.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 2 && args.length != 3) { System.err.printf("Usage: %s [generic options] <metadata_file> <output> [<mapper_classname>]\n", getClass().getSimpleName()); ToolRunner.printGenericCommandUsage(System.err); return -1; }/* w w w. ja v a 2 s . c om*/ String metadataFile = args[0]; Path outputPath = new Path(args[1]); Class<? extends Mapper> mapperClass = (args.length == 2) ? HAWQTableMapper.class : (Class<? extends Mapper>) Class.forName(args[2]); // delete previous output FileSystem fs = FileSystem.getLocal(getConf()); if (fs.exists(outputPath)) fs.delete(outputPath, true); fs.close(); Job job = new Job(getConf()); job.setJarByClass(MapReduceLocalDriver.class); job.setInputFormatClass(HAWQInputFormat.class); HAWQInputFormat.setInput(job.getConfiguration(), metadataFile); FileOutputFormat.setOutputPath(job, outputPath); job.setMapperClass(mapperClass); job.setReducerClass(HAWQTableReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); return job.waitForCompletion(true) ? 0 : 1; }