Example usage for org.apache.hadoop.fs Path Path

List of usage examples for org.apache.hadoop.fs Path Path

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:TestIndexServer.java

License:Open Source License

public void testTaskQueueRunner() throws Exception {
    status = initializeindexconf();//from w  w w  . j a v a2s  .  com
    fs.delete(new Path(status.gettablelocation()), true);
    fs.delete(new Path(status.gettablelocation()), true);
    UtilIndexStorage.writeFDF(status.gettablelocation(), 5, 10000, (short) -1, false, false, true, true);

    IndexServer server = new IndexServer(null);
    server.testmode = true;
    server.start();
    Thread.sleep(5000);
    server.close();

}

From source file:TestIndexServer.java

License:Open Source License

public void testServerUpdateIndex() throws Exception {
    status = initializeindexconf();//from w w  w . ja  v  a  2s.c  o  m
    fs.delete(new Path(status.gettablelocation()), true);
    fs.delete(new Path(status.gettablelocation()), true);
    UtilIndexStorage.writeFDF(status.gettablelocation(), 5, 10000, (short) -1, false, false, true, true);

    IndexServer server = new IndexServer(null);
    server.testmode = true;
    server.start();
    ArrayList<IndexItemStatus> itemstatuss = new ArrayList<IndexItemStatus>();
    while (true) {
        itemstatuss.clear();
        File file = new File("indexconf");
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        int num = dis.readInt();
        for (int i = 0; i < num; i++) {
            IndexItemStatus status = new IndexItemStatus();
            status.read(dis);
            itemstatuss.add(status);
        }
        dis.close();
        if (itemstatuss.get(0).status == 2) {
            break;
        }
        if (itemstatuss.get(0).status < 0) {
            server.close();
            assertTrue(false);
        }
        Thread.sleep(3000);
    }

    server.close();
}

From source file:TestIndexServer.java

License:Open Source License

public void testServerMergeIndex() throws Exception {
    Configuration conf = new Configuration();
    conf.set("se.indexer.IPM_merge_interval", "5000");
    IndexServer server = new IndexServer(conf);
    server.testmode = true;// www. ja  v  a2s  .  c om
    server.start();
    ArrayList<IndexItemStatus> itemstatuss = new ArrayList<IndexItemStatus>();
    for (int k = 0; k < 0; k++) {
        while (true) {
            itemstatuss.clear();
            File file = new File("indexconf");
            DataInputStream dis = new DataInputStream(new FileInputStream(file));
            int num = dis.readInt();
            for (int i = 0; i < num; i++) {
                IndexItemStatus status = new IndexItemStatus();
                status.read(dis);
                itemstatuss.add(status);
            }
            dis.close();
            if (itemstatuss.get(0).status == 2) {
                break;
            }
            if (itemstatuss.get(0).status < 0) {
                server.close();
                assertTrue(false);
            }
            Thread.sleep(3000);
        }
        UtilIndexStorage.writeFDF(status.gettablelocation(), 3, 10000, (short) -1, false, false, true, false);
        while (true) {
            itemstatuss.clear();
            File file = new File("indexconf");
            DataInputStream dis = new DataInputStream(new FileInputStream(file));
            int num = dis.readInt();
            for (int i = 0; i < num; i++) {
                IndexItemStatus status = new IndexItemStatus();
                status.read(dis);
                itemstatuss.add(status);
            }
            dis.close();
            if (itemstatuss.get(0).status == 1) {
                break;
            }
            Thread.sleep(3000);
        }
    }

    Thread.sleep(15000);
    while (true) {
        itemstatuss.clear();
        File file = new File("indexconf");
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        int num = dis.readInt();
        for (int i = 0; i < num; i++) {
            IndexItemStatus status = new IndexItemStatus();
            status.read(dis);
            itemstatuss.add(status);
        }
        dis.close();
        if (itemstatuss.get(0).status == 2) {
            break;
        }
        Thread.sleep(3000);
    }

    FileStatus[] fss = fs.listStatus(new Path(itemstatuss.get(0).indexlocation + "/nopart"));
    assertEquals(fss.length, 1);

    server.close();
}

From source file:ColumnStroageStabilityTest.java

License:Open Source License

static void doReadRand(int count) {
    try {/*from   w  ww  . ja v a 2 s  .c o  m*/

        Path path = new Path(fullPrefix);
        ArrayList<Short> vector = new ArrayList<Short>(10);
        vector.add((short) 3);
        vector.add((short) 6);
        vector.add((short) 4);
        vector.add((short) 2);
        vector.add((short) 0);
        vector.add((short) 5);
        vector.add((short) 1);

        Configuration conf = new Configuration();
        ColumnStorageClient client = new ColumnStorageClient(path, vector, conf);
        long begin = System.currentTimeMillis();

        for (int i = 0; i < count; i++) {
            int rand = (int) (Math.random() * count);
            Record record = client.getRecordByLine(rand);
            /*
             if (record == null)
             {
            String string = "record no:" + i + " return null";
            out.write(string.getBytes());
             }
                     
                    
             if (!judgeRecord(record))
             {
            String string = "record no:" + i + " value error";
            out.write(string.getBytes());
             }
             */
            if (i % (100 * 10000) == 0) {
                String string = "read seq " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                System.out.println(string);
            }
        }

        long end = System.currentTimeMillis();
        String string = "Read Seq over, count:" + count + ", delay:" + (long) ((end - begin) / 1000) + " s";
        System.out.println(string);
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }

}

From source file:ColumnStroageStabilityTest.java

License:Open Source License

static void doReadSeq(int count) {
    try {/* w  w w . j  ava  2  s  .  c o  m*/

        Path path = new Path(prefix);
        ArrayList<Short> vector = new ArrayList<Short>(10);

        File file = new File("testReadMassRecord.result");
        FileOutputStream out = new FileOutputStream(file);

        for (int i = 0; i < idxs.length; i++) {
            vector.add(Short.valueOf(idxs[i]));
        }

        Configuration conf = new Configuration();
        ColumnStorageClient client = new ColumnStorageClient(path, vector, conf);
        long begin = System.currentTimeMillis();

        for (int i = 0; i < count; i++) {
            Record record = client.getRecordByLine(i);

            if (record == null) {
                String string = "record no:" + i + " return null \n";
                out.write(string.getBytes());
            }

            Error error = new Error();
            if (!judgeRecord(record, error)) {
                String string = "record no:" + i + ": " + error.msg + "\n";
                out.write(string.getBytes());
            }

            if (i % (100 * 10000) == 0) {
                String string = "read seq " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s ";
                out.write(string.getBytes());
                out.write(new String("\n").getBytes());
            }
        }

        client.close();

        long end = System.currentTimeMillis();
        String string = "Read Seq over, count:" + count + ", delay:" + (long) ((end - begin) / 1000) + " s \n";
        out.write(string.getBytes());
        out.close();
        System.out.println(string);
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }

}

From source file:TweetCategorizer.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    // conf.addResource(new Path("../../env_vars"));

    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: TweetCategorizer <in> <out>");
        System.exit(2);// www .  ja va  2s . c  o m
    }

    // ----------------------------------------------------------
    //         READ FILTER FILE
    // ----------------------------------------------------------
    // Path pt=new Path("hdfs://pathTofile");
    //Path pt = new Path("../www/hfilters.json");
    String l;
    String line = "";
    //FileSystem fs = FileSystem.get(conf);
    BufferedReader br = new BufferedReader(new FileReader("../www/json/filters.json"));

    try {
        //BufferedReader br = new BufferedReader(new FileReader(fs.open(pt)));

        while ((l = br.readLine()) != null) {
            line += l;
            //System.out.println(line);
        }

    } finally {
        // you should close out the BufferedReader
        br.close();
    }
    // ----------------------------------------------------------
    //         PARSE JSON
    //http://stackoverflow.com/questions/6697147/json-iterate-through-jsonarray
    //http://juliusdavies.ca/json-simple-1.1.1-javadocs/org/json/simple/JSONObject.html
    // ----------------------------------------------------------
    JSONParser parser = new JSONParser();
    JSONObject jsonObject = (JSONObject) parser.parse(line);

    Set<String> filters = jsonObject.keySet();

    // inside each object there is a "name" field, get value and add to keyword_list
    for (String i : filters) {
        JSONObject objects = (JSONObject) jsonObject.get(i);
        String keyword = ((String) objects.get("name")).toLowerCase();
        TokenizerMapper.keyname_list.add(i);
        TokenizerMapper.keyword_list.add(keyword);
    }
    // ----------------------------------------------------------

    Job job = new Job(conf, "categorize tweets");
    job.setJarByClass(TweetCategorizer.class);
    job.setMapperClass(TokenizerMapper.class);
    // job.setCombinerClass(IntSumReducer.class);
    // job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:TestParascaleFileStatus.java

License:Apache License

public void testParascaleFileStatus() {
    final Path p = new Path("/foo/bar");
    final long filesize = 10;
    final ParascaleFileStatusMock parascaleFileStatus = new ParascaleFileStatusMock(filesize, false, 2,
            32 * 1024 * 1024, System.currentTimeMillis(), p);
    parascaleFileStatus.permissionString = "-rw-r-xr-- 1 parascale parascale 0 Sep  9 12:37 16:43 bar";

    assertEquals("invalid blocksize", 32 * 1024 * 1024, parascaleFileStatus.getBlockSize());
    assertEquals("invalid user", "parascale", parascaleFileStatus.getOwner());
    assertEquals("invalid group", "parascale", parascaleFileStatus.getGroup());
    assertEquals("invalid size of file - should be 10 as set in contructor", 10, parascaleFileStatus.getLen());
}

From source file:TestParascaleFileStatus.java

License:Apache License

public void testLoadPermissionInfo() {
    final Path p = new Path("/foo/bar");
    {/*from  w  w  w  .j a v  a2 s. c o m*/
        final ParascaleFileStatusMock parascaleFileStatus = new ParascaleFileStatusMock(10, false, 2,
                32 * 1024 * 1024, System.currentTimeMillis(), p);
        parascaleFileStatus.permissionString = "-rw-r-xr-- 1 parascale parascale 0 Sep  9 12:37 16:43 bar";
        final FsPermission permission = parascaleFileStatus.getPermission();
        assertEquals(FsAction.READ, permission.getOtherAction());
        assertEquals(FsAction.READ_EXECUTE, permission.getGroupAction());
        assertEquals(FsAction.READ_WRITE, permission.getUserAction());
    }
    {
        final ParascaleFileStatusMock parascaleFileStatus = new ParascaleFileStatusMock(10, false, 2,
                32 * 1024 * 1024, System.currentTimeMillis(), p);
        parascaleFileStatus.permissionString = "-rw--wxr-- 1 parascale parascale 0 Sep  9 12:37 16:43 bar";
        assertEquals(32 * 1024 * 1024, parascaleFileStatus.getBlockSize());
        assertEquals("parascale", parascaleFileStatus.getOwner());
        final FsPermission permission = parascaleFileStatus.getPermission();
        assertEquals(FsAction.READ, permission.getOtherAction());
        assertEquals(FsAction.WRITE_EXECUTE, permission.getGroupAction());
        assertEquals(FsAction.READ_WRITE, permission.getUserAction());

    }
    final ParascaleFileStatusMock parascaleFileStatus = new ParascaleFileStatusMock(10, false, 2,
            32 * 1024 * 1024, System.currentTimeMillis(), p);
    parascaleFileStatus.permissionString = "-rw-r-xr-- 1 parascale parascale 0 Sep  9 12:37 16:43 bar";
    assertEquals("permissions already loaded - should be lazy", 0, parascaleFileStatus.count.get());
    parascaleFileStatus.getPermission();
    assertEquals("permissions loaded more than once", 1, parascaleFileStatus.count.get());
    parascaleFileStatus.getOwner();
    assertEquals("permissions loaded more than once", 1, parascaleFileStatus.count.get());
    parascaleFileStatus.getGroup();
    assertEquals("permissions loaded more than once", 1, parascaleFileStatus.count.get());
}

From source file:WordLines.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 3) {
        System.err.println("Usage: wordlines <in> [<in>...] <SearchTerm> <out>");
        System.exit(2);/*from w ww  .j  a v  a 2s. c om*/
    }
    conf.set("searchWord", otherArgs[otherArgs.length - 2]);
    Job job = new Job(conf, "word lines");
    job.setJarByClass(WordLines.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 0; i < otherArgs.length - 2; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:DataJoinJob.java

License:Apache License

public static JobConf createDataJoinJob(String args[]) throws IOException {

    String inputDir = args[0];//from w  w w  . ja  va2  s.c o m
    String outputDir = args[1];
    Class inputFormat = SequenceFileInputFormat.class;
    if (args[2].compareToIgnoreCase("text") != 0) {
        System.out.println("Using SequenceFileInputFormat: " + args[2]);
    } else {
        System.out.println("Using TextInputFormat: " + args[2]);
        inputFormat = TextInputFormat.class;
    }
    int numOfReducers = Integer.parseInt(args[3]);
    Class mapper = getClassByName(args[4]);
    Class reducer = getClassByName(args[5]);
    Class mapoutputValueClass = getClassByName(args[6]);
    Class outputFormat = TextOutputFormat.class;
    Class outputValueClass = Text.class;
    if (args[7].compareToIgnoreCase("text") != 0) {
        System.out.println("Using SequenceFileOutputFormat: " + args[7]);
        outputFormat = SequenceFileOutputFormat.class;
        outputValueClass = getClassByName(args[7]);
    } else {
        System.out.println("Using TextOutputFormat: " + args[7]);
    }
    long maxNumOfValuesPerGroup = 100;
    String jobName = "";
    if (args.length > 8) {
        maxNumOfValuesPerGroup = Long.parseLong(args[8]);
    }
    if (args.length > 9) {
        jobName = args[9];
    }
    Configuration defaults = new Configuration();
    JobConf job = new JobConf(defaults, DataJoinJob.class);
    job.setJobName("DataJoinJob: " + jobName);

    FileSystem fs = FileSystem.get(defaults);
    fs.delete(new Path(outputDir));
    FileInputFormat.setInputPaths(job, inputDir);

    job.setInputFormat(inputFormat);

    job.setMapperClass(mapper);
    FileOutputFormat.setOutputPath(job, new Path(outputDir));
    job.setOutputFormat(outputFormat);
    SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(mapoutputValueClass);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(outputValueClass);
    job.setReducerClass(reducer);

    job.setNumMapTasks(1);
    job.setNumReduceTasks(numOfReducers);
    job.setLong("datajoin.maxNumOfValuesPerGroup", maxNumOfValuesPerGroup);
    return job;
}