List of usage examples for org.apache.hadoop.fs FileSystem isFile
@Deprecated public boolean isFile(Path f) throws IOException
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testSetNoPreFileName() { try {// w w w.ja va 2 s .c o m Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); Head head = new Head(); String fileName = prefix + "testSetNoPreFileName"; fd.create(fileName, head); Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); if (!fs.isFile(path)) { fail("create file fail"); } } catch (IOException e) { e.printStackTrace(); fail("get ioexception:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testSetPreFileName() { try {//from w w w .ja v a 2s .c o m Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); Head head = new Head(); //String fileName = "hdfs://tdw-172-25-38-246:54310/user/tdwadmin/testSetPreFileName"; String fileName = "/user/tdwadmin/se_test/fs/basic/testSetPreFileName"; fd.create(fileName, head); fd.close(); Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); if (!fs.isFile(path)) { fail("create file fail"); } } catch (IOException e) { fail(e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:adts.HbaseClient.java
License:Open Source License
public static void main(String[] args) throws IOException { String[] keys = new String[5]; int keywords_counter = 0; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path inFile = new Path(args[0]); if (!fs.exists(inFile)) System.out.println("Input file not found"); if (!fs.isFile(inFile)) System.out.println("Input should be a file"); else {/* ww w. ja va 2s . c o m*/ FSDataInputStream fsDataInputStream = fs.open(inFile); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fsDataInputStream)); String line; while (((line = bufferedReader.readLine()) != null) && (keywords_counter < 5)) { String[] array = line.split("\t"); String keyword = array[0]; System.out.println("Record : " + keyword); keys[keywords_counter] = keyword; keywords_counter++; } bufferedReader.close(); fs.close(); Configuration config = HBaseConfiguration.create(); HTable table = new HTable(config, "index"); Random randomGenerator = new Random(); for (int i = 0; i < 10; i++) { int randomInt = randomGenerator.nextInt(5); System.out.println("Random chosen keyword : " + keys[randomInt]); FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL); SingleColumnValueFilter filter_by_name = new SingleColumnValueFilter(Bytes.toBytes("keyword"), Bytes.toBytes(""), CompareOp.EQUAL, Bytes.toBytes(keys[randomInt])); //filter_by_name.setFilterIfMissing(true); list.addFilter(filter_by_name); Scan scan = new Scan(); scan.setFilter(list); //scan.addFamily(Bytes.toBytes("keyword")); ResultScanner scanner = table.getScanner(scan); try { for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { // print out the row we found and the columns we were looking for byte[] cells = rr.getValue(Bytes.toBytes("article"), Bytes.toBytes("")); System.out.println("Keyword " + keys[randomInt] + "belonging to article with md5 : " + Bytes.toString(cells)); } } catch (Exception e) { e.printStackTrace(); } finally { scanner.close(); } } table.close(); } }
From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCF.java
License:Apache License
@Override public boolean load(String path, boolean lazy) { this.m_isLazyLoadModel = lazy; this.m_modelPath = path; if (lazy == false) { Path dataPath = new Path(m_modelPath); Configuration conf = new Configuration(); try {/* w w w .j a va 2 s .c o m*/ FileSystem fs = dataPath.getFileSystem(conf); LinkedList<Path> files = new LinkedList<Path>(); if (!fs.exists(dataPath)) { this.m_isLazyLoadModel = false; this.m_modelPath = null; return false; } if (!fs.isFile(dataPath)) { for (int i = 0; i < 100000; i++) { Path partFile = new Path( m_modelPath + "/part-" + String.valueOf(100000 + i).substring(1, 6)); if (fs.exists(partFile)) { files.add(partFile); } else { break; } } } else { files.add(dataPath); } LOG.info("loading model from " + path); for (Path file : files) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); Text key = new Text(); PipesVectorWritable value = new PipesVectorWritable(); String strKey = null; Long actualKey = null; String firstSymbol = null; while (reader.next(key, value) != false) { strKey = key.toString(); firstSymbol = strKey.substring(0, 1); try { actualKey = Long.valueOf(strKey.substring(1)); } catch (Exception e) { actualKey = new Long(0); } if (firstSymbol.equals(OnlineCF.DFLT_MODEL_ITEM_DELIM)) { // LOG.info("loaded itemId: " + actualKey + " itemVector: " // + value.getVector()); m_modelItemFactorizedValues.put(actualKey, new PipesVectorWritable(value)); } else if (firstSymbol.equals(OnlineCF.DFLT_MODEL_USER_DELIM)) { // LOG.info("loaded userId: " + actualKey + " userVector: " // + value.getVector()); m_modelUserFactorizedValues.put(actualKey, new PipesVectorWritable(value)); } else { // unknown continue; } } reader.close(); } LOG.info("loaded: " + m_modelUserFactorizedValues.size() + " users, " + m_modelItemFactorizedValues.size() + " items"); // for (Long user : m_modelUserFactorizedValues.keySet()) { // LOG.info("userId: " + user + " userVector: " // + m_modelUserFactorizedValues.get(user)); // } // for (Long item : m_modelItemFactorizedValues.keySet()) { // LOG.info("itemId: " + item + " itemVector: " // + m_modelItemFactorizedValues.get(item)); // } } catch (Exception e) { e.printStackTrace(); this.m_isLazyLoadModel = false; this.m_modelPath = null; return false; } } return true; }
From source file:azkaban.crypto.Decryptions.java
License:Open Source License
public String decrypt(final String cipheredText, final String passphrasePath, final FileSystem fs) throws IOException { Preconditions.checkNotNull(cipheredText); Preconditions.checkNotNull(passphrasePath); final Path path = new Path(passphrasePath); Preconditions.checkArgument(fs.exists(path), "File does not exist at " + passphrasePath); Preconditions.checkArgument(fs.isFile(path), "Passphrase path is not a file. " + passphrasePath); final FileStatus fileStatus = fs.getFileStatus(path); Preconditions.checkArgument(USER_READ_PERMISSION_ONLY.equals(fileStatus.getPermission()), "Passphrase file should only have read only permission on only user. " + passphrasePath); final Crypto crypto = new Crypto(); try (BufferedReader br = new BufferedReader( new InputStreamReader(fs.open(path), Charset.defaultCharset()))) { final String passphrase = br.readLine(); final String decrypted = crypto.decrypt(cipheredText, passphrase); Preconditions.checkNotNull(decrypted, "Was not able to decrypt"); return decrypted; }//w w w. ja va2 s . c o m }
From source file:azkaban.jobtype.javautils.Whitelist.java
License:Open Source License
/** * Updates whitelist if there's any change. If it needs to update whitelist, it enforces writelock * to make sure//from w ww .j a v a 2s . c o m * there's an exclusive access on shared variables. */ @VisibleForTesting Set<String> retrieveWhitelist(FileSystem fs, Path path) { try { Preconditions.checkArgument(fs.exists(path), "File does not exist at " + path); Preconditions.checkArgument(fs.isFile(path), "Whitelist path is not a file. " + path); Set<String> result = Sets.newHashSet(); try (BufferedReader br = new BufferedReader( new InputStreamReader(fs.open(path), StandardCharsets.UTF_8))) { String s = null; while (!StringUtils.isEmpty((s = br.readLine()))) { result.add(s); } } return result; } catch (IOException e) { throw new RuntimeException(e); } }
From source file:azkaban.viewer.hdfs.HdfsBrowserServlet.java
License:Apache License
private void handleFsDisplay(String user, HttpServletRequest req, HttpServletResponse resp, Session session) throws IOException, ServletException, IllegalArgumentException, IllegalStateException { FileSystem fs = null; try {/* w w w.ja va 2 s. c o m*/ fs = getFileSystem(user); } catch (HadoopSecurityManagerException e) { errorPage(user, req, resp, session, "Cannot get FileSystem."); return; } Path path = getPath(req); if (logger.isDebugEnabled()) { logger.debug("path: '" + path.toString() + "'"); } try { if (!fs.exists(path)) { errorPage(user, req, resp, session, path.toUri().getPath() + " does not exist."); fs.close(); return; } } catch (IOException ioe) { logger.error("Got exception while checking for existence of path '" + path + "'", ioe); errorPage(user, req, resp, session, path.toUri().getPath() + " Encountered error while trying to detect if path '" + path + "' exists. Reason: " + ioe.getMessage()); fs.close(); return; } if (fs.isFile(path)) { displayFilePage(fs, user, req, resp, session, path); } else if (fs.getFileStatus(path).isDir()) { displayDirPage(fs, user, req, resp, session, path); } else { errorPage(user, req, resp, session, "It exists, it is not a file, and it is not a directory, what " + "is it precious?"); } fs.close(); }
From source file:be.ugent.intec.halvade.utils.HalvadeConf.java
License:Open Source License
public static void setKnownSitesOnHDFS(Configuration conf, String[] val) throws IOException, URISyntaxException { conf.setInt(numberOfSites, val.length); FileSystem fs; for (int i = 0; i < val.length; i++) { // check if dir add all files! fs = FileSystem.get(new URI(val[i]), conf); if (fs.isFile(new Path(val[i]))) { conf.set(sitesOnHDFSName + i, val[i]); } else {/*w w w . j av a 2 s . co m*/ FileStatus[] files = fs.listStatus(new Path(val[i])); for (FileStatus file : files) { if (!file.isDir()) { conf.set(sitesOnHDFSName + i, file.getPath().toString()); } } } } }
From source file:boa.datagen.MapFileGen.java
License:Apache License
public static void main(String[] args) throws Exception { if (SEQ_FILE_PATH.isEmpty()) { System.out.println("Missing path to sequence file. Please specify it in the properties file."); return;/*w ww . ja va2 s .co m*/ } String base = "hdfs://boa-njt/"; Configuration conf = new Configuration(); conf.set("fs.default.name", base); FileSystem fs = FileSystem.get(conf); Path path = new Path(SEQ_FILE_PATH); String name = path.getName(); if (fs.isFile(path)) { if (path.getName().equals(MapFile.DATA_FILE_NAME)) { MapFile.fix(fs, path.getParent(), Text.class, BytesWritable.class, false, conf); } else { Path dataFile = new Path(path.getParent(), MapFile.DATA_FILE_NAME); fs.rename(path, dataFile); Path dir = new Path(path.getParent(), name); fs.mkdirs(dir); fs.rename(dataFile, new Path(dir, dataFile.getName())); MapFile.fix(fs, dir, Text.class, BytesWritable.class, false, conf); } } else { FileStatus[] files = fs.listStatus(path); for (FileStatus file : files) { path = file.getPath(); if (fs.isFile(path)) { Path dataFile = new Path(path.getParent(), MapFile.DATA_FILE_NAME); fs.rename(path, dataFile); MapFile.fix(fs, dataFile.getParent(), Text.class, BytesWritable.class, false, conf); break; } } } fs.close(); }
From source file:br.ufpr.inf.hpath.HPath.java
License:Apache License
/** * Execute the XPath query as a Hadoop job * @param xpath_query XPath query submitted by the user via cli. * @param inputFile XML file which has all data. * @param outputFile Query's result is stored in this file. * @throws Exception/*from w w w . ja va 2s . com*/ */ public static void main(String[] args) throws Exception { if (args.length < 1) { System.out.println("USAGE: hpath [xpath_query] [input_file] [<output_dir>]"); System.exit(-1); } System.out.println("***************"); System.out.println(" Query -> " + args[2]); System.out.println(" Input -> " + args[0]); System.out.println(" Output -> " + args[1]); System.out.println("***************"); String xpath_query = args[2]; String inputFile = args[0]; String outputFile = args[1]; String tag = ""; // tag = getFisrtQueryTag(xpath_query); tag = getLastQueryTag(xpath_query); Configuration conf = new Configuration(); conf.set("xmlinput.start", "<" + tag); conf.set("xmlinput.end", "</" + tag + ">"); conf.set("xpath.query", xpath_query); @SuppressWarnings("deprecation") Job job = new Job(conf, "HPath"); FileSystem fs = FileSystem.get(conf); Path inFile = new Path(inputFile); Path outFile = new Path(outputFile); if (!fs.exists(inFile)) { System.out.println("error: Input file not found."); System.exit(-1); } if (!fs.isFile(inFile)) { System.out.println("error: Input should be a file."); System.exit(-1); } if (fs.exists(outFile)) { System.out.println("error: Output already exists."); System.exit(-1); } job.setJarByClass(HPath.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(XmlItemInputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, inFile); FileOutputFormat.setOutputPath(job, outFile); job.waitForCompletion(true); }