List of usage examples for org.apache.hadoop.fs FSDataInputStream FSDataInputStream
public FSDataInputStream(InputStream in)
From source file:com.knewton.mrtool.io.JsonRecordReaderTest.java
License:Apache License
/** * Tests the line reader in the record reader to see if records can be read correctly from a * random seek location in the input stream. * /* w w w .j av a 2 s . c o m*/ * @throws IOException * @throws InterruptedException */ @Test public void testJsonRecordReaderWithRandomPos() throws IOException, InterruptedException { JsonRecordReader<Text> rr = new JsonRecordReader<Text>() { @Override protected Class<?> getDataClass(String jsonStr) { return Text.class; } }; Configuration conf = new Configuration(); TaskAttemptContext context = new TaskAttemptContext(conf, new TaskAttemptID()); FileSplit fileSplit = new FileSplit(new Path("recs.2013-03-20_02_52.log"), 10, recommendationBytes.length, new String[0]); new MockUp<FileSystem>() { @Mock public FSDataInputStream open(Path f) throws IOException { return new FSDataInputStream(new SeekableByteArrayInputStream(recommendationBytes)); } }; // Initialize it to get the compression codecs rr.initialize(fileSplit, context); // close the line reader and reopen it. rr.close(); LineReader lineReader = rr.initLineReader(fileSplit, conf); Text line = new Text(); lineReader.readLine(line); assertEquals(DummyJsonRecommendations.jsonRecommendations[1], line.toString()); line = new Text(); lineReader.readLine(line); assertTrue(line.toString().isEmpty()); lineReader.close(); }
From source file:com.netflix.bdp.inviso.history.TraceService.java
License:Apache License
/** * Returns a json object representing the job history. * * @param jobId//from w w w . ja va 2 s.com * @param path Use the given path as opposed to the history locator * @param summary Return just the top level details of the job * @param counters Include counters * @return Json string * @throws Exception */ @Path("load/{jobId}") @GET @Produces("application/json") public String trace(@PathParam("jobId") final String jobId, @QueryParam("path") final String path, @QueryParam("summary") boolean summary, @QueryParam("counters") @DefaultValue("true") boolean counters) throws Exception { Pair<org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path> historyPath; if (path != null) { historyPath = new ImmutablePair<>(null, new org.apache.hadoop.fs.Path(path)); } else { historyPath = historyLocator.locate(jobId); } if (historyPath == null) { throw new WebApplicationException(404); } TraceJobHistoryLoader loader = new TraceJobHistoryLoader(properties); FileSystem fs = FileSystem.get(historyPath.getRight().toUri(), config); CompressionCodec codec = new CompressionCodecFactory(config).getCodec(historyPath.getRight()); FSDataInputStream fin = fs.open(historyPath.getRight()); if (codec != null) { fin = new FSDataInputStream(new WrappedCompressionInputStream(codec.createInputStream(fin))); } JobHistoryParser parser = new JobHistoryParser(fin); parser.parse(loader); String[] ignore = { "counters" }; ObjectMapper mapper = new ObjectMapper(); SimpleModule module = new SimpleModule("MyModule", new Version(1, 0, 0, null)); //Job JavaType jobMapType = MapLikeType.construct(Job.class, SimpleType.construct(String.class), SimpleType.construct(Object.class)); module.addSerializer(Job.class, MapSerializer.construct(ignore, jobMapType, false, null, null, null, null)); //Task JavaType taskMapType = MapLikeType.construct(Task.class, SimpleType.construct(String.class), SimpleType.construct(Object.class)); module.addSerializer(Task.class, MapSerializer.construct(ignore, taskMapType, false, null, null, null, null)); //Attempt JavaType attemptMapType = MapLikeType.construct(TaskAttempt.class, SimpleType.construct(String.class), SimpleType.construct(Object.class)); module.addSerializer(TaskAttempt.class, MapSerializer.construct(ignore, attemptMapType, false, null, null, null, null)); if (!counters) { mapper.registerModule(module); } if (summary) { loader.getJob().clearTasks(); } return mapper.writeValueAsString(loader.getJob()); }
From source file:com.pigai.hadoop.HttpFSFileSystem.java
License:Apache License
/** * Opens an FSDataInputStream at the indicated Path. </p> IMPORTANT: the * returned <code><FSDataInputStream/code> does not support the * <code>PositionReadable</code> and <code>Seekable</code> methods. * // ww w. ja v a2 s .c om * @param f * the file name to open * @param bufferSize * the size of the buffer to be used. */ @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put(OP_PARAM, Operation.OPEN.toString()); HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params, f, true); validateResponse(conn, HttpURLConnection.HTTP_OK); return new FSDataInputStream(new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); }
From source file:com.quantcast.qfs.hadoop.QFSImpl.java
License:Apache License
public FSDataInputStream open(String path, int bufferSize) throws IOException { return new FSDataInputStream(createQFSInputStream(kfsAccess, path, statistics)); }
From source file:com.qubole.rubix.core.CachingFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { FSDataInputStream inputStream = fs.open(path, bufferSize); if (skipCache(path, getConf())) { cacheSkipped = true;// w ww . j a va2s .c om return inputStream; } return new FSDataInputStream(new BufferedFSInputStream( new CachingInputStream(inputStream, this, path, this.getConf(), statsMBean, clusterManager.getSplitSize(), clusterManager.getClusterType()), CacheConfig.getBlockSize(getConf()))); }
From source file:com.qubole.rubix.core.TestCachingInputStream.java
License:Apache License
public void createCachingStream(Configuration conf) throws InterruptedException, IOException { conf.setBoolean(CacheConfig.DATA_CACHE_STRICT_MODE, true); conf.setInt(CacheConfig.dataCacheBookkeeperPortConf, 3456); File file = new File(backendFileName); LocalFSInputStream localFSInputStream = new LocalFSInputStream(backendFileName); FSDataInputStream fsDataInputStream = new FSDataInputStream(localFSInputStream); conf.setInt(CacheConfig.blockSizeConf, blockSize); log.info("All set to test"); // This should be after server comes up else client could not be created inputStream = new CachingInputStream(fsDataInputStream, conf, backendPath, file.length(), file.lastModified(), new CachingFileSystemStats(), 64 * 1024 * 1024, ClusterType.TEST_CLUSTER_MANAGER); }
From source file:com.qubole.rubix.core.TestRemoteReadRequestChain.java
License:Apache License
@BeforeMethod public void setup() throws IOException { // Populate File DataGen.populateFile(backendFileName); LocalFSInputStream localFSInputStream = new LocalFSInputStream(backendFileName); fsDataInputStream = new FSDataInputStream(localFSInputStream); remoteReadRequestChain = new RemoteReadRequestChain(fsDataInputStream, localFileName); }
From source file:com.quixey.hadoop.fs.oss.OSSFileSystem.java
License:Apache License
@Override @Nonnull//w w w .j av a 2 s .c o m public FSDataInputStream open(Path path, int bufferSize) throws IOException { path = checkNotNull(path); FileStatus fs = getFileStatus(path); // will throw if the file doesn't exist if (fs.isDirectory()) throw new FileNotFoundException("'" + path + "' is a directory"); LOG.info("Opening '{}' for reading", path); Path absolutePath = makeAbsolute(path); String key = pathToKey(absolutePath); return new FSDataInputStream( new BufferedFSInputStream(new OSSFileInputStream(store, key, of(statistics)), bufferSize)); }
From source file:com.rapleaf.ramhdfs.RamFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { if (!exists(f)) { throw new FileNotFoundException(f.toString()); }//from w ww. j ava 2s . c o m return new FSDataInputStream( new BufferedFSInputStream(new RamFSInputStream(pathToFileObject(f)), bufferSize)); }
From source file:com.ricemap.spateDB.core.RTree.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { // Tree size (Header + structure + data) treeSize = in.readInt();/*from www . j av a2 s .com*/ if (treeSize == 0) { height = elementCount = 0; return; } // Read only the tree structure in memory while actual records remain on // disk and loaded when necessary height = in.readInt(); if (height == 0) return; degree = in.readInt(); elementCount = in.readInt(); columnar = in.readInt() == 1; // Keep only tree structure in memory nodeCount = (int) ((powInt(degree, height) - 1) / (degree - 1)); int structureSize = nodeCount * NodeSize; byte[] treeStructure = new byte[structureSize]; in.readFully(treeStructure, 0, structureSize); structure = new FSDataInputStream(new MemoryInputStream(treeStructure)); if (in instanceof FSDataInputStream) { this.treeStartOffset = ((FSDataInputStream) in).getPos() - structureSize - TreeHeaderSize; this.data = (FSDataInputStream) in; } else { // Load all tree data in memory this.treeStartOffset = 0 - structureSize - TreeHeaderSize; int treeDataSize = treeSize - TreeHeaderSize - structureSize; byte[] treeData = new byte[treeDataSize]; in.readFully(treeData, 0, treeDataSize); this.data = new FSDataInputStream(new MemoryInputStream(treeData)); } nodeCount = (int) ((Math.pow(degree, height) - 1) / (degree - 1)); leafNodeCount = (int) Math.pow(degree, height - 1); nonLeafNodeCount = nodeCount - leafNodeCount; }