List of usage examples for org.apache.hadoop.fs FileSystem getConf
@Override
public Configuration getConf()
From source file:nthu.scopelab.tsqr.ssvd.SSVDSolver.java
License:Apache License
/** * helper capabiltiy to load distributed row matrices into dense matrix (to * support tests mainly)./*from w w w. java2s .c om*/ * * @param fs * filesystem * @param glob * FS glob * @param conf * configuration * @return Dense matrix array * @throws IOException * when I/O occurs. */ public static double[][] loadDistributedRowMatrix(FileSystem fs, Path glob, Configuration conf) throws IOException { FileStatus[] files = fs.globStatus(glob); if (files == null) { return null; } List<double[]> denseData = new ArrayList<double[]>(); /* * assume it is partitioned output, so we need to read them up in order of * partitions. */ Arrays.sort(files, PARTITION_COMPARATOR); SequenceFile.Reader reader = null; IntWritable key = null; VectorWritable value = null; for (FileStatus fstat : files) { reader = new SequenceFile.Reader(fs, fstat.getPath(), fs.getConf()); try { key = (IntWritable) reader.getKeyClass().newInstance(); value = (VectorWritable) reader.getValueClass().newInstance(); } catch (Exception e) { e.printStackTrace(); } while (reader.next(key, value)) { Vector v = value.get(); int size = v.size(); double[] row = new double[size]; for (int i = 0; i < size; i++) { row[i] = v.get(i); } // ignore row label. denseData.add(row); } } if (reader != null) reader.close(); return denseData.toArray(new double[denseData.size()][]); }
From source file:nthu.scopelab.tsqr.ssvd.SSVDSolver.java
License:Apache License
/** * Load multiplel upper triangular matrices and sum them up. * /*from w ww. jav a2s . c o m*/ * @param fs * @param glob * @param conf * @return the sum of upper triangular inputs. * @throws IOException */ public static cmUpperTriangDenseMatrix loadAndSumUpperTriangMatrices(FileSystem fs, Path glob, Configuration conf) throws IOException { FileStatus[] files = fs.globStatus(glob); if (files == null) { return null; } /* * assume it is partitioned output, so we need to read them up in order of * partitions. */ Arrays.sort(files, PARTITION_COMPARATOR); DenseVector result = null; SequenceFile.Reader reader = null; Writable rkey; IntWritable key = null; VectorWritable value = null; for (FileStatus fstat : files) { reader = new SequenceFile.Reader(fs, fstat.getPath(), fs.getConf()); try { key = (IntWritable) reader.getKeyClass().newInstance(); value = (VectorWritable) reader.getValueClass().newInstance(); } catch (Exception e) { e.printStackTrace(); } while (reader.next(key, value)) { Vector v = value.get(); if (result == null) { result = new DenseVector(v); } else { result.add(v); } } } if (reader != null) reader.close(); if (result == null) { throw new IOException("Unexpected underrun in upper triangular matrix files"); } return new cmUpperTriangDenseMatrix(result.getData()); }
From source file:org.apache.accumulo.core.client.impl.BulkImport.java
License:Apache License
public static Map<KeyExtent, Long> estimateSizes(AccumuloConfiguration acuConf, Path mapFile, long fileSize, Collection<KeyExtent> extents, FileSystem ns) throws IOException { long totalIndexEntries = 0; Map<KeyExtent, MLong> counts = new TreeMap<>(); for (KeyExtent keyExtent : extents) counts.put(keyExtent, new MLong(0)); Text row = new Text(); FileSKVIterator index = FileOperations.getInstance().newIndexReaderBuilder() .forFile(mapFile.toString(), ns, ns.getConf()).withTableConfiguration(acuConf).build(); try {/*from w w w . ja v a 2 s . c o m*/ while (index.hasTop()) { Key key = index.getTopKey(); totalIndexEntries++; key.getRow(row); // TODO this could use a binary search for (Entry<KeyExtent, MLong> entry : counts.entrySet()) if (entry.getKey().contains(row)) entry.getValue().l++; index.next(); } } finally { try { if (index != null) index.close(); } catch (IOException e) { log.debug("Failed to close " + mapFile, e); } } Map<KeyExtent, Long> results = new TreeMap<>(); for (KeyExtent keyExtent : extents) { double numEntries = counts.get(keyExtent).l; if (numEntries == 0) numEntries = 1; long estSize = (long) ((numEntries / totalIndexEntries) * fileSize); results.put(keyExtent, estSize); } return results; }
From source file:org.apache.accumulo.core.client.impl.BulkImport.java
License:Apache License
public static List<KeyExtent> findOverlappingTablets(ClientContext context, KeyExtentCache extentCache, Path file, FileSystem fs) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder() .forFile(file.toString(), fs, fs.getConf()).withTableConfiguration(context.getConfiguration()) .seekToBeginning().build()) { return findOverlappingTablets(context, extentCache, null, null, reader); }/* www. ja v a 2 s . c om*/ }
From source file:org.apache.accumulo.core.client.mock.MockTableOperations.java
License:Apache License
@Override public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { long time = System.currentTimeMillis(); MockTable table = acu.tables.get(tableName); if (table == null) { throw new TableNotFoundException(null, tableName, "The table was not found"); }/*ww w .j av a2 s . c o m*/ Path importPath = new Path(dir); Path failurePath = new Path(failureDir); FileSystem fs = acu.getFileSystem(); /* * check preconditions */ // directories are directories if (fs.isFile(importPath)) { throw new IOException("Import path must be a directory."); } if (fs.isFile(failurePath)) { throw new IOException("Failure path must be a directory."); } // failures are writable Path createPath = failurePath.suffix("/.createFile"); FSDataOutputStream createStream = null; try { createStream = fs.create(createPath); } catch (IOException e) { throw new IOException("Error path is not writable."); } finally { if (createStream != null) { createStream.close(); } } fs.delete(createPath, false); // failures are empty FileStatus[] failureChildStats = fs.listStatus(failurePath); if (failureChildStats.length > 0) { throw new IOException("Error path must be empty."); } /* * Begin the import - iterate the files in the path */ for (FileStatus importStatus : fs.listStatus(importPath)) { try { FileSKVIterator importIterator = FileOperations.getInstance().newReaderBuilder() .forFile(importStatus.getPath().toString(), fs, fs.getConf()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).seekToBeginning() .build(); while (importIterator.hasTop()) { Key key = importIterator.getTopKey(); Value value = importIterator.getTopValue(); if (setTime) { key.setTimestamp(time); } Mutation mutation = new Mutation(key.getRow()); if (!key.isDeleted()) { mutation.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp(), value); } else { mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp()); } table.addMutation(mutation); importIterator.next(); } } catch (Exception e) { FSDataOutputStream failureWriter = null; DataInputStream failureReader = null; try { failureWriter = fs.create(failurePath.suffix("/" + importStatus.getPath().getName())); failureReader = fs.open(importStatus.getPath()); int read = 0; byte[] buffer = new byte[1024]; while (-1 != (read = failureReader.read(buffer))) { failureWriter.write(buffer, 0, read); } } finally { if (failureReader != null) failureReader.close(); if (failureWriter != null) failureWriter.close(); } } fs.delete(importStatus.getPath(), true); } }
From source file:org.apache.accumulo.core.client.mock.MockTableOperationsImpl.java
License:Apache License
@Override public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { long time = System.currentTimeMillis(); MockTable table = acu.tables.get(tableName); if (table == null) { throw new TableNotFoundException(null, tableName, "The table was not found"); }/* w w w. j av a2 s . c om*/ Path importPath = new Path(dir); Path failurePath = new Path(failureDir); FileSystem fs = acu.getFileSystem(); /* * check preconditions */ // directories are directories if (fs.isFile(importPath)) { throw new IOException("Import path must be a directory."); } if (fs.isFile(failurePath)) { throw new IOException("Failure path must be a directory."); } // failures are writable Path createPath = failurePath.suffix("/.createFile"); FSDataOutputStream createStream = null; try { createStream = fs.create(createPath); } catch (IOException e) { throw new IOException("Error path is not writable."); } finally { if (createStream != null) { createStream.close(); } } fs.delete(createPath, false); // failures are empty FileStatus[] failureChildStats = fs.listStatus(failurePath); if (failureChildStats.length > 0) { throw new IOException("Error path must be empty."); } /* * Begin the import - iterate the files in the path */ for (FileStatus importStatus : fs.listStatus(importPath)) { try { FileSKVIterator importIterator = FileOperations.getInstance().openReader( importStatus.getPath().toString(), true, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration()); while (importIterator.hasTop()) { Key key = importIterator.getTopKey(); Value value = importIterator.getTopValue(); if (setTime) { key.setTimestamp(time); } Mutation mutation = new Mutation(key.getRow()); if (!key.isDeleted()) { mutation.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp(), value); } else { mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp()); } table.addMutation(mutation); importIterator.next(); } } catch (Exception e) { FSDataOutputStream failureWriter = null; DataInputStream failureReader = null; try { failureWriter = fs.create(failurePath.suffix("/" + importStatus.getPath().getName())); failureReader = fs.open(importStatus.getPath()); int read = 0; byte[] buffer = new byte[1024]; while (-1 != (read = failureReader.read(buffer))) { failureWriter.write(buffer, 0, read); } } finally { if (failureReader != null) failureReader.close(); if (failureWriter != null) failureWriter.close(); } } fs.delete(importStatus.getPath(), true); } }
From source file:org.apache.accumulo.core.clientImpl.bulk.BulkImport.java
License:Apache License
public static Map<KeyExtent, Long> estimateSizes(AccumuloConfiguration acuConf, Path mapFile, long fileSize, Collection<KeyExtent> extents, FileSystem ns, Cache<String, Long> fileLenCache, CryptoService cs) throws IOException { if (extents.size() == 1) { return Collections.singletonMap(extents.iterator().next(), fileSize); }/*from ww w . j av a2 s . c o m*/ long totalIndexEntries = 0; Map<KeyExtent, MLong> counts = new TreeMap<>(); for (KeyExtent keyExtent : extents) counts.put(keyExtent, new MLong(0)); Text row = new Text(); FileSKVIterator index = FileOperations.getInstance().newIndexReaderBuilder() .forFile(mapFile.toString(), ns, ns.getConf(), cs).withTableConfiguration(acuConf) .withFileLenCache(fileLenCache).build(); try { while (index.hasTop()) { Key key = index.getTopKey(); totalIndexEntries++; key.getRow(row); // TODO this could use a binary search for (Entry<KeyExtent, MLong> entry : counts.entrySet()) if (entry.getKey().contains(row)) entry.getValue().l++; index.next(); } } finally { try { if (index != null) index.close(); } catch (IOException e) { log.debug("Failed to close " + mapFile, e); } } Map<KeyExtent, Long> results = new TreeMap<>(); for (KeyExtent keyExtent : extents) { double numEntries = counts.get(keyExtent).l; if (numEntries == 0) numEntries = 1; long estSize = (long) ((numEntries / totalIndexEntries) * fileSize); results.put(keyExtent, estSize); } return results; }
From source file:org.apache.accumulo.core.clientImpl.bulk.BulkImport.java
License:Apache License
public static List<KeyExtent> findOverlappingTablets(ClientContext context, KeyExtentCache extentCache, Path file, FileSystem fs, Cache<String, Long> fileLenCache, CryptoService cs) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder() .forFile(file.toString(), fs, fs.getConf(), cs).withTableConfiguration(context.getConfiguration()) .withFileLenCache(fileLenCache).seekToBeginning().build()) { return findOverlappingTablets(extentCache, reader); }// w w w . j a v a2 s. c o m }
From source file:org.apache.accumulo.core.clientImpl.BulkImport.java
License:Apache License
public static List<KeyExtent> findOverlappingTablets(ClientContext context, KeyExtentCache extentCache, Path file, FileSystem fs, Cache<String, Long> fileLenCache, CryptoService cs) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder() .forFile(file.toString(), fs, fs.getConf(), cs).withTableConfiguration(context.getConfiguration()) .withFileLenCache(fileLenCache).seekToBeginning().build()) { return findOverlappingTablets(context, extentCache, reader); }/*from w w w . j a va 2s . c o m*/ }
From source file:org.apache.accumulo.proxy.SimpleProxyBase.java
License:Apache License
@Test public void testTableOperations() throws Exception { final String TABLE_TEST = makeTableName(); client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS); // constraints client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName()); assertEquals(2, client.listConstraints(creds, TABLE_TEST).size()); UtilWaitThread.sleep(2000);/*from w w w . j av a2s.c om*/ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "123")); try { client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x")); fail("constraint did not fire"); } catch (MutationsRejectedException ex) { } client.removeConstraint(creds, TABLE_TEST, 2); UtilWaitThread.sleep(2000); assertEquals(1, client.listConstraints(creds, TABLE_TEST).size()); client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x")); assertScan(new String[][] { { "row1", "cf", "cq", "x" } }, TABLE_TEST); // splits, merge client.addSplits(creds, TABLE_TEST, new HashSet<ByteBuffer>(Arrays.asList(s2bb("a"), s2bb("m"), s2bb("z")))); List<ByteBuffer> splits = client.listSplits(creds, TABLE_TEST, 1); assertEquals(Arrays.asList(s2bb("m")), splits); client.mergeTablets(creds, TABLE_TEST, null, s2bb("m")); splits = client.listSplits(creds, TABLE_TEST, 10); assertEquals(Arrays.asList(s2bb("m"), s2bb("z")), splits); client.mergeTablets(creds, TABLE_TEST, null, null); splits = client.listSplits(creds, TABLE_TEST, 10); List<ByteBuffer> empty = Collections.emptyList(); assertEquals(empty, splits); // iterators client.deleteTable(creds, TABLE_TEST); client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS); HashMap<String, String> options = new HashMap<String, String>(); options.put("type", "STRING"); options.put("columns", "cf"); IteratorSetting setting = new IteratorSetting(10, TABLE_TEST, SummingCombiner.class.getName(), options); client.attachIterator(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class)); for (int i = 0; i < 10; i++) { client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "1")); } assertScan(new String[][] { { "row1", "cf", "cq", "10" } }, TABLE_TEST); try { client.checkIteratorConflicts(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class)); fail("checkIteratorConflicts did not throw an exception"); } catch (Exception ex) { } client.deleteRows(creds, TABLE_TEST, null, null); client.removeIterator(creds, TABLE_TEST, "test", EnumSet.allOf(IteratorScope.class)); String expected[][] = new String[10][]; for (int i = 0; i < 10; i++) { client.updateAndFlush(creds, TABLE_TEST, mutation("row" + i, "cf", "cq", "" + i)); expected[i] = new String[] { "row" + i, "cf", "cq", "" + i }; client.flushTable(creds, TABLE_TEST, null, null, true); } assertScan(expected, TABLE_TEST); // clone final String TABLE_TEST2 = makeTableName(); client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null); assertScan(expected, TABLE_TEST2); client.deleteTable(creds, TABLE_TEST2); // don't know how to test this, call it just for fun client.clearLocatorCache(creds, TABLE_TEST); // compact client.compactTable(creds, TABLE_TEST, null, null, null, true, true, null); assertEquals(1, countFiles(TABLE_TEST)); assertScan(expected, TABLE_TEST); // get disk usage client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null); Set<String> tablesToScan = new HashSet<String>(); tablesToScan.add(TABLE_TEST); tablesToScan.add(TABLE_TEST2); tablesToScan.add("foo"); client.createTable(creds, "foo", true, TimeType.MILLIS); List<DiskUsage> diskUsage = (client.getDiskUsage(creds, tablesToScan)); assertEquals(2, diskUsage.size()); assertEquals(1, diskUsage.get(0).getTables().size()); assertEquals(2, diskUsage.get(1).getTables().size()); client.compactTable(creds, TABLE_TEST2, null, null, null, true, true, null); diskUsage = (client.getDiskUsage(creds, tablesToScan)); assertEquals(3, diskUsage.size()); assertEquals(1, diskUsage.get(0).getTables().size()); assertEquals(1, diskUsage.get(1).getTables().size()); assertEquals(1, diskUsage.get(2).getTables().size()); client.deleteTable(creds, "foo"); client.deleteTable(creds, TABLE_TEST2); // export/import File dir = tempFolder.newFolder("test"); File destDir = tempFolder.newFolder("test_dest"); client.offlineTable(creds, TABLE_TEST, false); client.exportTable(creds, TABLE_TEST, dir.getAbsolutePath()); // copy files to a new location FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream is = fs.open(new Path(dir + "/distcp.txt")); BufferedReader r = new BufferedReader(new InputStreamReader(is)); while (true) { String line = r.readLine(); if (line == null) break; Path srcPath = new Path(line); FileUtils.copyFile(new File(srcPath.toUri().getPath()), new File(destDir, srcPath.getName())); } client.deleteTable(creds, TABLE_TEST); client.importTable(creds, "testify", destDir.getAbsolutePath()); assertScan(expected, "testify"); client.deleteTable(creds, "testify"); try { // ACCUMULO-1558 a second import from the same dir should fail, the first import moved the files client.importTable(creds, "testify2", destDir.getAbsolutePath()); fail(); } catch (Exception e) { } assertFalse(client.listTables(creds).contains("testify2")); // Locality groups client.createTable(creds, "test", true, TimeType.MILLIS); Map<String, Set<String>> groups = new HashMap<String, Set<String>>(); groups.put("group1", Collections.singleton("cf1")); groups.put("group2", Collections.singleton("cf2")); client.setLocalityGroups(creds, "test", groups); assertEquals(groups, client.getLocalityGroups(creds, "test")); // table properties Map<String, String> orig = client.getTableProperties(creds, "test"); client.setTableProperty(creds, "test", "table.split.threshold", "500M"); Map<String, String> update = client.getTableProperties(creds, "test"); assertEquals(update.get("table.split.threshold"), "500M"); client.removeTableProperty(creds, "test", "table.split.threshold"); update = client.getTableProperties(creds, "test"); assertEquals(orig, update); // rename table Map<String, String> tables = client.tableIdMap(creds); client.renameTable(creds, "test", "bar"); Map<String, String> tables2 = client.tableIdMap(creds); assertEquals(tables.get("test"), tables2.get("bar")); // table exists assertTrue(client.tableExists(creds, "bar")); assertFalse(client.tableExists(creds, "test")); // bulk import String filename = dir + "/bulk/import/rfile.rf"; FileSKVWriter writer = FileOperations.getInstance().openWriter(filename, fs, fs.getConf(), DefaultConfiguration.getInstance()); writer.startDefaultLocalityGroup(); writer.append(new org.apache.accumulo.core.data.Key(new Text("a"), new Text("b"), new Text("c")), new Value("value".getBytes())); writer.close(); fs.mkdirs(new Path(dir + "/bulk/fail")); client.importDirectory(creds, "bar", dir + "/bulk/import", dir + "/bulk/fail", true); String scanner = client.createScanner(creds, "bar", null); ScanResult more = client.nextK(scanner, 100); client.closeScanner(scanner); assertEquals(1, more.results.size()); ByteBuffer maxRow = client.getMaxRow(creds, "bar", null, null, false, null, false); assertEquals(s2bb("a"), maxRow); assertFalse(client.testTableClassLoad(creds, "bar", "abc123", SortedKeyValueIterator.class.getName())); assertTrue(client.testTableClassLoad(creds, "bar", VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName())); }