List of usage examples for org.apache.hadoop.fs FileSystem newInstance
public static FileSystem newInstance(URI uri, Configuration config) throws IOException
From source file:apex.benchmark.RedisHelper.java
License:Apache License
public void fillDB(String fileName) throws IOException { Path filePath = new Path(fileName); Configuration configuration = new Configuration(); FileSystem fs;//from w w w . j a va 2s . com fs = FileSystem.newInstance(filePath.toUri(), configuration); FSDataInputStream inputStream = fs.open(filePath); BufferedReader bufferedReader; try { bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); String line; while ((line = bufferedReader.readLine()) != null) { String[] mapping = line.split("\\s+"); if (mapping.length != 2) { continue; } jedis.sadd("campaigns", mapping[0]); jedis.set(mapping[1], mapping[0]); } } catch (Exception e) { throw e; } }
From source file:co.cask.tephra.persist.CommitMarkerCodecTest.java
License:Apache License
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration hConf = new Configuration(); hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath()); dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build(); conf = new Configuration(dfsCluster.getFileSystem().getConf()); fs = FileSystem.newInstance(FileSystem.getDefaultUri(conf), conf); }
From source file:co.cask.tephra.persist.HDFSTransactionLogTest.java
License:Apache License
private void testTransactionLogSync(int totalCount, int batchSize, boolean withMarker, boolean isComplete) throws Exception { List<TransactionEdit> edits = TransactionEditUtil.createRandomEdits(totalCount); long timestamp = System.currentTimeMillis(); Configuration configuration = getConfiguration(); FileSystem fs = FileSystem.newInstance(FileSystem.getDefaultUri(configuration), configuration); SequenceFile.Writer writer = getSequenceFileWriter(configuration, fs, timestamp, withMarker); AtomicLong logSequence = new AtomicLong(); HDFSTransactionLog transactionLog = getHDFSTransactionLog(configuration, fs, timestamp); AbstractTransactionLog.Entry entry;/*from w w w . j av a 2s . co m*/ for (int i = 0; i < totalCount - batchSize; i += batchSize) { if (withMarker) { writeNumWrites(writer, batchSize); } for (int j = 0; j < batchSize; j++) { entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(j)); writer.append(entry.getKey(), entry.getEdit()); } writer.syncFs(); } if (withMarker) { writeNumWrites(writer, batchSize); } for (int i = totalCount - batchSize; i < totalCount - 1; i++) { entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(i)); writer.append(entry.getKey(), entry.getEdit()); } entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(totalCount - 1)); if (isComplete) { writer.append(entry.getKey(), entry.getEdit()); } else { byte[] bytes = Longs.toByteArray(entry.getKey().get()); writer.appendRaw(bytes, 0, bytes.length, new SequenceFile.ValueBytes() { @Override public void writeUncompressedBytes(DataOutputStream outStream) throws IOException { byte[] test = new byte[] { 0x2 }; outStream.write(test, 0, 1); } @Override public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException { // no-op } @Override public int getSize() { // mimic size longer than the actual byte array size written, so we would reach EOF return 12; } }); } writer.syncFs(); Closeables.closeQuietly(writer); // now let's try to read this log TransactionLogReader reader = transactionLog.getReader(); int syncedEdits = 0; while (reader.next() != null) { // testing reading the transaction edits syncedEdits++; } if (isComplete) { Assert.assertEquals(totalCount, syncedEdits); } else { Assert.assertEquals(totalCount - batchSize, syncedEdits); } }
From source file:co.cask.tephra.persist.HDFSTransactionStateStorage.java
License:Apache License
@Override protected void startUp() throws Exception { Preconditions.checkState(configuredSnapshotDir != null, "Snapshot directory is not configured. Please set " + TxConstants.Manager.CFG_TX_SNAPSHOT_DIR + " in configuration."); String hdfsUser = hConf.get(TxConstants.Manager.CFG_TX_HDFS_USER); if (hdfsUser == null || UserGroupInformation.isSecurityEnabled()) { if (hdfsUser != null && LOG.isDebugEnabled()) { LOG.debug("Ignoring configuration {}={}, running on secure Hadoop", TxConstants.Manager.CFG_TX_HDFS_USER, hdfsUser); }/* www .ja va 2 s. c om*/ // NOTE: we can start multiple times this storage. As hdfs uses per-jvm cache, we want to create new fs instead // of getting closed one fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf); } else { fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf, hdfsUser); } snapshotDir = new Path(configuredSnapshotDir); LOG.info("Using snapshot dir " + snapshotDir); }
From source file:com.datatorrent.contrib.enrich.FSLoader.java
License:Apache License
@Override public void connect() throws IOException { Configuration conf = new Configuration(); this.filePath = new Path(fileName); this.fs = FileSystem.newInstance(filePath.toUri(), conf); if (!fs.isFile(filePath)) { throw new IOException("Provided path " + fileName + " is not a file"); }/* w ww .j a va2 s. c o m*/ connected = true; }
From source file:com.datatorrent.contrib.hdht.HDHTFileAccessFSImpl.java
License:Open Source License
@Override public void init() { if (fs == null) { Path dataFilePath = new Path(basePath); try {// ww w . ja v a2s . co m fs = FileSystem.newInstance(dataFilePath.toUri(), new Configuration()); } catch (IOException e) { DTThrowable.rethrow(e); } } }
From source file:com.datatorrent.contrib.hive.HiveOperator.java
License:Apache License
/** * Override this method to change the FileSystem instance that is used by the * operator./* ww w .j av a 2s . c o m*/ * * @return A FileSystem object. * @throws IOException */ protected FileSystem getHDFSInstance() throws IOException { FileSystem tempFS = FileSystem.newInstance(new Path(store.filepath).toUri(), new Configuration()); if (!tempFS.getScheme().equalsIgnoreCase("hdfs")) { localString = " local"; } return tempFS; }
From source file:com.datatorrent.flume.source.HdfsTestSource.java
License:Open Source License
@Override public void configure(Context context) { directory = context.getString(SOURCE_DIR); rate = context.getInteger(RATE, rate); initDate = context.getString(INIT_DATE); Preconditions.checkArgument(!Strings.isNullOrEmpty(directory)); directoryPath = new Path(directory); String[] parts = initDate.split("-"); Preconditions.checkArgument(parts.length == 3); Calendar calendar = Calendar.getInstance(); calendar.set(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]) - 1, Integer.parseInt(parts[2]), 0, 0, 0);/*ww w . java 2 s . c o m*/ initTime = calendar.getTimeInMillis(); try { List<String> files = findFiles(); for (String file : files) { dataFiles.add(file); } if (logger.isDebugEnabled()) { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); logger.debug("settings {} {} {} {} {}", directory, rate, dateFormat.format(oneDayBack), dateFormat.format(new Date(initTime)), currentFile); for (String file : dataFiles) { logger.debug("settings add file {}", file); } } fs = FileSystem.newInstance(new Path(directory).toUri(), configuration); Path filePath = new Path(dataFiles.get(currentFile)); br = new BufferedReader(new InputStreamReader(new GzipCompressorInputStream(fs.open(filePath)))); } catch (IOException e) { throw new RuntimeException(e); } finished = true; }
From source file:com.datatorrent.flume.source.HdfsTestSource.java
License:Open Source License
private List<String> findFiles() throws IOException { List<String> files = Lists.newArrayList(); Path directoryPath = new Path(directory); FileSystem lfs = FileSystem.newInstance(directoryPath.toUri(), configuration); try {//from w w w. j av a2 s .c o m logger.debug("checking for new files in {}", directoryPath); RemoteIterator<LocatedFileStatus> statuses = lfs.listFiles(directoryPath, true); for (; statuses.hasNext();) { FileStatus status = statuses.next(); Path path = status.getPath(); String filePathStr = path.toString(); if (!filePathStr.endsWith(".gz")) { continue; } logger.debug("new file {}", filePathStr); files.add(path.toString()); } } catch (FileNotFoundException e) { logger.warn("Failed to list directory {}", directoryPath, e); throw new RuntimeException(e); } finally { lfs.close(); } return files; }
From source file:com.datatorrent.lib.bucket.BucketManagerTest.java
License:Open Source License
@AfterClass public static void teardown() throws IOException { manager.shutdownService();//ww w.j a va2s. co m Path root = new Path(applicationPath); FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration()); fs.delete(root, true); }