List of usage examples for org.apache.hadoop.fs FileSystem makeQualified
public Path makeQualified(Path path)
From source file:org.apache.accumulo.server.fs.VolumeManagerImpl.java
License:Apache License
@Override public Path getFullPath(FileType fileType, String path) { int colon = path.indexOf(':'); if (colon > -1) { // Check if this is really an absolute path or if this is a 1.4 style relative path for a WAL if (fileType == FileType.WAL && path.charAt(colon + 1) != '/') { path = path.substring(path.indexOf('/')); } else {/*from w w w. ja v a 2 s . com*/ return new Path(path); } } if (path.startsWith("/")) path = path.substring(1); // ACCUMULO-2974 To ensure that a proper absolute path is created, the caller needs to include the table ID // in the relative path. Fail when this doesn't appear to happen. if (FileType.TABLE == fileType) { // Trailing slash doesn't create an additional element String[] pathComponents = StringUtils.split(path, Path.SEPARATOR_CHAR); // Is an rfile if (path.endsWith(RFILE_SUFFIX)) { if (pathComponents.length < 3) { throw new IllegalArgumentException("Fewer components in file path than expected"); } } else { // is a directory if (pathComponents.length < 2) { throw new IllegalArgumentException("Fewer components in directory path than expected"); } } } // normalize the path Path fullPath = new Path(defaultVolume.getBasePath(), fileType.getDirectory()); fullPath = new Path(fullPath, path); FileSystem fs = getVolumeByPath(fullPath).getFileSystem(); return fs.makeQualified(fullPath); }
From source file:org.apache.accumulo.server.util.ZooKeeperMain.java
License:Apache License
@Override public void execute(final String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(ZooKeeperMain.class.getName(), args); FileSystem fs = VolumeManagerImpl.get().getDefaultVolume().getFileSystem(); String baseDir = ServerConstants.getBaseUris()[0]; System.out.println(/*from w w w . j a va2 s.com*/ "Using " + fs.makeQualified(new Path(baseDir + "/instance_id")) + " to lookup accumulo instance"); Instance instance = HdfsZooInstance.getInstance(); if (opts.servers == null) { opts.servers = instance.getZooKeepers(); } System.out.println("The accumulo instance id is " + instance.getInstanceID()); if (!opts.servers.contains("/")) opts.servers += "/accumulo/" + instance.getInstanceID(); org.apache.zookeeper.ZooKeeperMain .main(new String[] { "-server", opts.servers, "-timeout", "" + (opts.timeout * 1000) }); }
From source file:org.apache.accumulo.test.BulkImportSequentialRowsIT.java
License:Apache License
@Test public void testBulkImportFailure() throws Exception { String tableName = getUniqueNames(1)[0]; TableOperations to = getConnector().tableOperations(); to.create(tableName);//from w w w.java 2 s . co m FileSystem fs = getFileSystem(); Path rootPath = new Path(fs.makeQualified(getUsableDir()), getClass().getSimpleName()); log.info("Writing to {}", rootPath); if (fs.exists(rootPath)) { assertTrue(fs.delete(rootPath, true)); } assertTrue(fs.mkdirs(rootPath)); Path bulk = new Path(rootPath, "bulk"); log.info("bulk: {}", bulk); assertTrue(fs.mkdirs(bulk)); Path err = new Path(rootPath, "err"); log.info("err: {}", err); assertTrue(fs.mkdirs(bulk)); assertTrue(fs.mkdirs(err)); Path rfile = new Path(bulk, "file.rf"); log.info("Generating RFile {}", rfile.toUri().toString()); GenerateSequentialRFile.main(new String[] { "-f", rfile.toUri().toString(), "-nr", Long.toString(NR), "-nv", Long.toString(NV) }); assertTrue("Expected that " + rfile + " exists, but it does not", fs.exists(rfile)); FsShell fsShell = new FsShell(fs.getConf()); assertEquals("Failed to chmod " + rootPath, 0, fsShell.run(new String[] { "-chmod", "-R", "777", rootPath.toString() })); // Add some splits to.addSplits(tableName, getSplits()); // Then import a single rfile to all the tablets, hoping that we get a failure to import because of the balancer moving tablets around // and then we get to verify that the bug is actually fixed. to.importDirectory(tableName, bulk.toString(), err.toString(), false); // The bug is that some tablets don't get imported into. assertEquals(NR * NV, Iterables.size(getConnector().createScanner(tableName, Authorizations.EMPTY))); }
From source file:org.apache.accumulo.test.functional.BulkFailureIT.java
License:Apache License
/** * This test verifies two things. First it ensures that after a bulk imported file is compacted * that import request are ignored. Second it ensures that after the bulk import transaction is * canceled that import request fail. The public API for bulk import can not be used for this * test. Internal (non public API) RPCs and Zookeeper state is manipulated directly. This is the * only way to interleave compactions with multiple, duplicate import RPC request. *//*from w ww .j a v a 2s. c om*/ protected void runTest(String table, long fateTxid, Loader loader) throws IOException, AccumuloException, AccumuloSecurityException, TableExistsException, KeeperException, InterruptedException, Exception, FileNotFoundException, TableNotFoundException { try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { SortedMap<Key, Value> testData = createTestData(); FileSystem fs = getCluster().getFileSystem(); String testFile = createTestFile(fateTxid, testData, fs); c.tableOperations().create(table); String tableId = c.tableOperations().tableIdMap().get(table); // Table has no splits, so this extent corresponds to the tables single tablet KeyExtent extent = new KeyExtent(TableId.of(tableId), null, null); ServerContext asCtx = getServerContext(); ZooArbitrator.start(asCtx, Constants.BULK_ARBITRATOR_TYPE, fateTxid); VolumeManager vm = asCtx.getVolumeManager(); // move the file into a directory for the table and rename the file to something unique String bulkDir = BulkImport.prepareBulkImport(asCtx, vm, testFile, TableId.of(tableId)); // determine the files new name and path FileStatus status = fs.listStatus(new Path(bulkDir))[0]; Path bulkLoadPath = fs.makeQualified(status.getPath()); // Directly ask the tablet to load the file. loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), false); assertEquals(ImmutableSet.of(bulkLoadPath), getFiles(c, extent)); assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent)); assertEquals(testData, readTable(table, c)); // Compact the bulk imported file. Subsequent request to load the file should be ignored. c.tableOperations().compact(table, new CompactionConfig().setWait(true)); Set<Path> tabletFiles = getFiles(c, extent); assertFalse(tabletFiles.contains(bulkLoadPath)); assertEquals(1, tabletFiles.size()); assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent)); assertEquals(testData, readTable(table, c)); // this request should be ignored by the tablet loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), false); assertEquals(tabletFiles, getFiles(c, extent)); assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent)); assertEquals(testData, readTable(table, c)); // this is done to ensure the tablet reads the load flags from the metadata table when it // loads c.tableOperations().offline(table, true); c.tableOperations().online(table, true); // this request should be ignored by the tablet loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), false); assertEquals(tabletFiles, getFiles(c, extent)); assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent)); assertEquals(testData, readTable(table, c)); // After this, all load request should fail. ZooArbitrator.stop(asCtx, Constants.BULK_ARBITRATOR_TYPE, fateTxid); c.securityOperations().grantTablePermission(c.whoami(), MetadataTable.NAME, TablePermission.WRITE); BatchDeleter bd = c.createBatchDeleter(MetadataTable.NAME, Authorizations.EMPTY, 1); bd.setRanges(Collections.singleton(extent.toMetadataRange())); bd.fetchColumnFamily(BulkFileColumnFamily.NAME); bd.delete(); loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), true); assertEquals(tabletFiles, getFiles(c, extent)); assertEquals(ImmutableSet.of(), getLoaded(c, extent)); assertEquals(testData, readTable(table, c)); } }
From source file:org.apache.accumulo.test.functional.BulkFailureIT.java
License:Apache License
private String createTestFile(long txid, SortedMap<Key, Value> testData, FileSystem fs) throws IOException { Path base = new Path(getCluster().getTemporaryPath(), "testBulk_ICI_" + txid); fs.delete(base, true);/*from w ww .j a v a2s . c o m*/ fs.mkdirs(base); Path files = new Path(base, "files"); try (RFileWriter writer = RFile.newWriter().to(new Path(files, "ici_01.rf").toString()).withFileSystem(fs) .build()) { writer.append(testData.entrySet()); } String filesStr = fs.makeQualified(files).toString(); return filesStr; }
From source file:org.apache.accumulo.tserver.log.MultiReader.java
License:Apache License
public MultiReader(VolumeManager fs, Path directory) throws IOException { boolean foundFinish = false; for (FileStatus child : fs.listStatus(directory)) { if (child.getPath().getName().startsWith("_")) continue; if (SortedLogState.isFinished(child.getPath().getName())) { foundFinish = true;//ww w. j av a 2s .c o m continue; } FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem(); heap.add(new Index(new Reader(ns.makeQualified(child.getPath()), ns.getConf()))); } if (!foundFinish) throw new IOException( "Sort \"" + SortedLogState.FINISHED.getMarker() + "\" flag not found in " + directory); }
From source file:org.apache.accumulo.tserver.log.MultiReaderTest.java
License:Apache License
@Before public void setUp() throws Exception { root.create();//from www. j a v a2 s. com String path = root.getRoot().getAbsolutePath() + "/manyMaps"; fs = VolumeManagerImpl.getLocal(path); Path root = new Path("file://" + path); fs.mkdirs(root); fs.create(new Path(root, "finished")).close(); FileSystem ns = fs.getVolumeByPath(root).getFileSystem(); Writer oddWriter = new Writer(ns.getConf(), ns.makeQualified(new Path(root, "odd")), Writer.keyClass(IntWritable.class), Writer.valueClass(BytesWritable.class)); BytesWritable value = new BytesWritable("someValue".getBytes()); for (int i = 1; i < 1000; i += 2) { oddWriter.append(new IntWritable(i), value); } oddWriter.close(); Writer evenWriter = new Writer(ns.getConf(), ns.makeQualified(new Path(root, "even")), Writer.keyClass(IntWritable.class), Writer.valueClass(BytesWritable.class)); for (int i = 0; i < 1000; i += 2) { if (i == 10) continue; evenWriter.append(new IntWritable(i), value); } evenWriter.close(); }
From source file:org.apache.accumulo.tserver.log.RecoveryLogReader.java
License:Apache License
public RecoveryLogReader(VolumeManager fs, Path directory, LogFileKey start, LogFileKey end) throws IOException { boolean foundFinish = false; for (FileStatus child : fs.listStatus(directory)) { if (child.getPath().getName().startsWith("_")) continue; if (SortedLogState.isFinished(child.getPath().getName())) { foundFinish = true;/*from www . j a v a2 s . c om*/ continue; } FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem(); heap.add(new Index(new Reader(ns.makeQualified(child.getPath()), ns.getConf()))); } if (!foundFinish) throw new IOException( "Sort \"" + SortedLogState.FINISHED.getMarker() + "\" flag not found in " + directory); iter = new SortCheckIterator(new RangeIterator(start, end)); }
From source file:org.apache.blur.spark.BlurMRBulkLoadSparkProcessor.java
License:Apache License
@Override protected Function2<JavaPairRDD<String, RowMutation>, Time, Void> getFunction() { return new Function2<JavaPairRDD<String, RowMutation>, Time, Void>() { @Override/*from w w w . j a v a2s. c o m*/ public Void call(JavaPairRDD<String, RowMutation> rdd, Time time) throws Exception { // Blur Table Details Iface client = getBlurClient(); TableDescriptor tableDescriptor = client.describe(getBlurTableName()); Configuration conf = new Configuration(); // Blur specific Configuration conf.setClass(MAPREDUCE_PARTITIONER_CLASS, BlurPartitioner.class, Partitioner.class); conf.set(MAPRED_OUTPUT_COMMITTER_CLASS, BlurOutputCommitter.class.getName()); // Partition RDD to match Blur Table Shard Count. Used Custom // Partitioner to channel correct BlurMutate to correct Shard. BlurSparkPartitioner blurSparkPartitioner = new BlurSparkPartitioner( tableDescriptor.getShardCount()); JavaPairRDD<Text, BlurMutate> flatMapToPair = rdd .flatMapToPair(new PairFlatMapFunction<Tuple2<String, RowMutation>, Text, BlurMutate>() { @Override public Iterable<Tuple2<Text, BlurMutate>> call(Tuple2<String, RowMutation> tuple2) throws Exception { RowMutation rowMutation = tuple2._2; final List<BlurMutate> result = new ArrayList<BlurMutate>(); List<RecordMutation> recordMutations = rowMutation.getRecordMutations(); String rowId = rowMutation.getRowId(); for (RecordMutation recordMutation : recordMutations) { Record record = recordMutation.getRecord(); String family = record.getFamily(); String recordId = record.getRecordId(); List<BlurColumn> columns = toColumns(record.getColumns()); BlurRecord blurRecord = new BlurRecord(); blurRecord.setRowId(rowId); blurRecord.setFamily(family); blurRecord.setRecordId(recordId); blurRecord.setColumns(columns); result.add(new BlurMutate(MUTATE_TYPE.REPLACE, blurRecord)); } return new Iterable<Tuple2<Text, BlurMutate>>() { @Override public Iterator<Tuple2<Text, BlurMutate>> iterator() { final Iterator<BlurMutate> iterator = result.iterator(); return new Iterator<Tuple2<Text, BlurMutate>>() { @Override public boolean hasNext() { return iterator.hasNext(); } @Override public Tuple2<Text, BlurMutate> next() { BlurMutate blurMutate = iterator.next(); return new Tuple2<Text, BlurMutate>( new Text(blurMutate.getRecord().getRowId()), blurMutate); } @Override public void remove() { } }; } }; } private List<BlurColumn> toColumns(List<Column> columns) { List<BlurColumn> cols = new ArrayList<BlurColumn>(); for (Column column : columns) { cols.add(new BlurColumn(column.getName(), column.getValue())); } return cols; } }); final JavaPairRDD<Text, BlurMutate> pRdd = flatMapToPair.partitionBy(blurSparkPartitioner) .persist(getStorageLevel()); Job job = new Job(conf); BlurOutputFormat.setupJob(job, tableDescriptor); Path path = new Path(getOutputPath()); FileSystem fileSystem = path.getFileSystem(conf); Path qualified = fileSystem.makeQualified(path); BlurOutputFormat.setOutputPath(job, qualified); setupBlurHadoopConfig(job.getConfiguration()); // Write the RDD to Blur Table if (pRdd.count() > 0) { pRdd.saveAsNewAPIHadoopFile(tableDescriptor.getTableUri(), Text.class, BlurMutate.class, BlurOutputFormat.class, job.getConfiguration()); client.loadData(getBlurTableName(), qualified.toString()); } return null; } }; }
From source file:org.apache.crunch.io.impl.FileTargetImpl.java
License:Apache License
protected static boolean isCompatible(FileSystem fs, Path path) { try {/*from w w w. j ava2 s . c o m*/ fs.makeQualified(path); return true; } catch (IllegalArgumentException e) { return false; } }