Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException 

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.linkedin.cubert.plan.physical.JobExecutor.java

License:Open Source License

protected void setLibjars() throws IOException {
    if (!root.has("libjars"))
        return;//w  ww.j  ava  2  s .  c o  m

    FileSystem localFs = FileSystem.getLocal(conf);

    for (JsonNode node : asArray(root, "libjars")) {
        Path path = new Path(node.getTextValue());

        if (localFs.exists(path)) {
            Path dstPath = new Path(tmpDir, path.getName());
            fs.copyFromLocalFile(path, dstPath);

            path = dstPath;
        }

        DistributedCache.addFileToClassPath(path, conf, fs);

    }
}

---

From source file:com.linkedin.mapred.AvroDistributedCacheFileReader.java

License:Open Source License

@Override
protected FileSystem getFilesystem(JobConf conf, Path path) throws IOException {
    return FileSystem.getLocal(new Configuration());
}

---

From source file:com.liveramp.cascading_ext.bloom.BloomFilterOperation.java

License:Apache License

@Override
public void prepare(FlowProcess flowProcess, OperationCall operationCall) {
    if (filter == null || !filterJobId.equals(jobId)) {
        try {/*from  w  w w .j  av a2 s .c  o m*/
            LOG.info("Loading bloom filter");

            String bloomFilter = getBloomFilterFile((JobConf) flowProcess.getConfigCopy());
            filter = BloomFilter.read(FileSystem.getLocal(new Configuration()), new Path(bloomFilter));
            filterJobId = jobId;

            LOG.info("Done loading bloom filter");
        } catch (Exception e) {
            throw new RuntimeException("Error loading bloom filter", e);
        }
    }
}

---

From source file:com.liveramp.cascading_ext.FileSystemHelper.java

License:Apache License

/**
 * merge all files in <code>sourceDir</code> into local <code>targetFile</code>, retrying on failure
 *//*w  w  w  . j av a  2s  . com*/
public static void copyMergeToLocal(String srcDir, String dstFile, int numTries, long delayBetweenTries)
        throws IOException {
    Configuration conf = new Configuration();
    FileSystem hdfs = getFS();
    FileSystem localfs = FileSystem.getLocal(conf);

    while (numTries-- > 0) {
        if (FileUtil.copyMerge(hdfs, new Path(srcDir), localfs, new Path(dstFile), false, conf, null)) {
            return;
        }
        try {
            Thread.sleep(delayBetweenTries);
        } catch (InterruptedException ie) {
            throw new RuntimeException(ie);
        }
    }
    throw new IOException("Could not copyMerge from \"" + srcDir + "\" to \"" + dstFile + "\"!");
}

---

From source file:com.mellanox.r4h.MiniDFSCluster.java

License:Apache License

public static void copyNameDirs(Collection<URI> srcDirs, Collection<URI> dstDirs, Configuration dstConf)
        throws IOException {
    URI srcDir = Lists.newArrayList(srcDirs).get(0);
    FileSystem dstFS = FileSystem.getLocal(dstConf).getRaw();
    for (URI dstDir : dstDirs) {
        Preconditions.checkArgument(!dstDir.equals(srcDir), "src and dst are the same: " + dstDir);
        File dstDirF = new File(dstDir);
        if (dstDirF.exists()) {
            if (!FileUtil.fullyDelete(dstDirF)) {
                throw new IOException("Unable to delete: " + dstDirF);
            }/*from w w w. j ava 2s. c  om*/
        }
        LOG.info("Copying namedir from primary node dir " + srcDir + " to " + dstDir);
        FileUtil.copy(new File(srcDir), dstFS, new Path(dstDir), false, dstConf);
    }
}

---

From source file:com.metamx.druid.indexer.hadoop.FSSpideringIteratorTest.java

License:Open Source License

@Test
public void testIterator() {
    String[] testFiles = { "file1", "file2", "file3", "file4", "file5" };

    File baseDir = Files.createTempDir();

    try {/*  w  ww  . j a  va2s. co  m*/
        new File(baseDir, "dir1").mkdir();
        new File(baseDir, "dir1/file1").createNewFile();
        new File(baseDir, "dir1/file2").createNewFile();

        new File(baseDir, "dir2/subDir1").mkdirs();
        new File(baseDir, "dir2/subDir1/file3").createNewFile();
        new File(baseDir, "dir2/subDir2").mkdirs();
        new File(baseDir, "dir2/subDir2/file4").createNewFile();
        new File(baseDir, "dir2/subDir2/file5").createNewFile();

        List<String> files = Lists.newArrayList(
                Iterables.transform(FSSpideringIterator.spiderIterable(FileSystem.getLocal(new Configuration()),
                        new Path(baseDir.toString())), new Function<FileStatus, String>() {
                            @Override
                            public String apply(@Nullable FileStatus input) {
                                return input.getPath().getName();
                            }
                        }));

        for (int i = 0; i < testFiles.length; i++) {
            Assert.assertTrue(files.remove(testFiles[i]));
        }

        Assert.assertTrue(files.isEmpty());
    } catch (IOException e) {
        Throwables.propagate(e);
    } finally {
        try {
            FileUtils.deleteDirectory(baseDir);
        } catch (IOException e) {
            Throwables.propagate(e);
        }
    }
}

---

From source file:com.metamx.milano.ProtoTestObjects.java

License:Apache License

public ProtoTestObjects() {
    Configuration conf = new Configuration(true);
    try {//from  w  w w.j  av  a  2  s  .  c  om
        fs = FileSystem.getLocal(conf);
    } catch (IOException e) {
        IOUtils.closeQuietly(fs);
        throw Throwables.propagate(e);
    }

    workingPath = new Path(fs.getWorkingDirectory(), "test-temporary-files");
    try {
        fs.mkdirs(workingPath);
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }

    conf.set("mapred.input.dir", workingPath.toString());
    conf.set("mapred.output.dir", workingPath.toString());
    conf.set("mapred.output.key.class", Map.class.getName());
    conf.set("mapred.output.value.class", Text.class.getName());

    conf.set("milano.proto.builder", Testing.TestItem.class.getName());

    context = new TaskAttemptContext(conf, new TaskAttemptID());

    HashMap<String, Object> testHash0 = new HashMap<String, Object>();

    testHash0.put("id", 1);
    testHash0.put("name", "Test 1");
    testHash0.put("type", "FOO");
    testHash0.put("data", Arrays.asList("Datum 1", "Datum 2"));

    HashMap<String, Object> itemMap0 = new HashMap<String, Object>();
    itemMap0.put("id", 42);
    itemMap0.put("name", "Foo");
    testHash0.put("item", itemMap0);

    Testing.TestItem testItem0 = Testing.TestItem.newBuilder().setId(1).setName("Test 1")
            .setType(Testing.TestItem.Type.FOO).addData("Datum 1").addData("Datum 2")
            .setItem(Testing.TestItem.SubItem.newBuilder().setId(42).setName("Foo").build()).build();

    Assert.assertTrue(testItem0.isInitialized());

    byte[] testItemBytes0 = testItem0.toByteArray();
    Testing.TestItem newItem0 = null;
    try {
        newItem0 = Testing.TestItem.parseFrom(testItemBytes0);
    } catch (InvalidProtocolBufferException e) {
        throw Throwables.propagate(e);
    }

    Assert.assertEquals(testItem0, newItem0);

    testItems.add(0, testItem0);
    testHashes.add(0, testHash0);
    testItemBytes.add(0, testItemBytes0);

    HashMap<String, Object> testHash1 = new HashMap<String, Object>();

    testHash1.put("id", 2);
    testHash1.put("name", "Test 2");
    testHash1.put("type", "BAR");
    testHash1.put("data", Arrays.asList("Datum 3", "Datum 4"));

    HashMap<String, Object> itemMap1 = new HashMap<String, Object>();
    itemMap1.put("id", 24);
    itemMap1.put("name", "Bar");
    testHash1.put("item", itemMap1);

    Testing.TestItem testItem1 = Testing.TestItem.newBuilder().setId(2).setName("Test 2")
            .setType(Testing.TestItem.Type.BAR).addData("Datum 3").addData("Datum 4")
            .setItem(Testing.TestItem.SubItem.newBuilder().setId(24).setName("Bar").build()).build();

    Assert.assertTrue(testItem1.isInitialized());

    byte[] testItemBytes1 = testItem1.toByteArray();
    Testing.TestItem newItem1 = null;
    try {
        newItem1 = Testing.TestItem.parseFrom(testItemBytes1);
    } catch (InvalidProtocolBufferException e) {
        throw Throwables.propagate(e);
    }

    Assert.assertEquals(testItem1, newItem1);

    testItems.add(1, testItem1);
    testHashes.add(1, testHash1);
    testItemBytes.add(1, testItemBytes1);

    HashMap<String, Object> testHash2 = new HashMap<String, Object>();

    testHash2.put("id", 2);
    testHash2.put("name", "Test 3");
    testHash2.put("type", "BAZ");
    testHash2.put("data", Arrays.asList("Datum 5", "Datum 6"));

    HashMap<String, Object> itemMap2 = new HashMap<String, Object>();
    itemMap2.put("id", 37);
    itemMap2.put("name", "Baz");
    testHash2.put("item", itemMap2);

    Testing.TestItem testItem2 = Testing.TestItem.newBuilder().setId(2).setName("Test 3")
            .setType(Testing.TestItem.Type.BAZ).addData("Datum 5").addData("Datum 6")
            .setItem(Testing.TestItem.SubItem.newBuilder().setId(37).setName("Baz").build()).build();

    Assert.assertTrue(testItem2.isInitialized());

    byte[] testItemBytes2 = testItem2.toByteArray();
    Testing.TestItem newItem2 = null;
    try {
        newItem2 = Testing.TestItem.parseFrom(testItemBytes2);
    } catch (InvalidProtocolBufferException e) {
        throw Throwables.propagate(e);
    }

    Assert.assertEquals(testItem2, newItem2);

    testItems.add(2, testItem2);
    testHashes.add(2, testHash2);
    testItemBytes.add(2, testItemBytes2);
}

---

From source file:com.moz.fiji.mapreduce.IntegrationTestFijiTableInputFormat.java

License:Apache License

public Job setupJob(String jobName, Path outputFile, Class<? extends Mapper> mapperClass,
        Class<? extends Reducer> reducerClass, EntityId startKey, EntityId limitKey, FijiRowFilter filter)
        throws Exception {
    final Job job = new Job(createConfiguration());
    final Configuration conf = job.getConfiguration();

    // Get settings for test.
    final FijiDataRequest request = FijiDataRequest.builder()
            .addColumns(ColumnsDef.create().add("info", "name").add("info", "email")).build();

    job.setJarByClass(IntegrationTestFijiTableInputFormat.class);

    // Setup the InputFormat.
    FijiTableInputFormat.configureJob(job, getFooTable().getURI(), request, startKey, limitKey, filter);
    job.setInputFormatClass(HBaseFijiTableInputFormat.class);

    // Duplicate functionality from MapReduceJobBuilder, since we are not using it here:
    final List<Path> jarFiles = Lists.newArrayList();
    final FileSystem fs = FileSystem.getLocal(conf);
    for (String cpEntry : System.getProperty("java.class.path").split(":")) {
        if (cpEntry.endsWith(".jar")) {
            jarFiles.add(fs.makeQualified(new Path(cpEntry)));
        }//from ww  w  .ja  va  2  s.  co  m
    }
    DistributedCacheJars.addJarsToDistributedCache(job, jarFiles);

    // Create a test job.
    job.setJobName(jobName);

    // Setup the OutputFormat.
    TextOutputFormat.setOutputPath(job, outputFile.getParent());
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    // Set the mapper class.
    if (null != mapperClass) {
        job.setMapperClass(mapperClass);
    }
    // Set the reducer class.
    if (null != reducerClass) {
        job.setReducerClass(reducerClass);
    }

    return job;
}

---

From source file:com.moz.fiji.mapreduce.kvstore.lib.FileStoreHelper.java

License:Apache License

/**
 * Deserializes file- and DistributedCache-specific properties associated
 * with the KeyValueStore that owns this FileStoreHelper from the specified configuration.
 *
 * <p>This retains a reference to the KeyValueStoreConfiguration's backing Configuration
 * instance to use when opening files specified by this configuration.</p>
 *
 * @param conf the configuration to read.
 * @throws IOException if there's an error deserializing the configuration.
 *///from  w w w. j ava2s. c o  m
public void initFromConf(KeyValueStoreConfiguration conf) throws IOException {
    setConf(conf.getDelegate());

    mDCachePrefix = conf.get(CONF_DCACHE_PREFIX_KEY, "");
    LOG.debug("Input dCachePrefix: " + mDCachePrefix);
    if (mDCachePrefix.isEmpty()) {
        // Read an ordinary list of files from the Configuration.
        // These may include directories and globs to expand.
        mInputPaths = Lists.map(Arrays.asList(conf.getStrings(CONF_PATHS_KEY, new String[0])),
                new Lists.Func<String, Path>() {
                    @Override
                    public Path eval(String in) {
                        LOG.debug("File input: " + in);
                        return new Path(in);
                    }
                });
    } else {
        // Use the dcache prefix to get the names of the files for this store.
        // The symlinks are already present in the working dir of the task.
        final FileSystem localFs = FileSystem.getLocal(conf.getDelegate());
        FileStatus[] statuses = localFs.globStatus(new Path(mDCachePrefix + "-*"));
        if (null == statuses || statuses.length == 0) {
            throw new IOException("No files associated with the job in the DistributedCache");
        }

        // Get the (absolute) input file paths to use.
        mInputPaths = Lists.map(Arrays.asList(statuses), new Lists.Func<FileStatus, Path>() {
            @Override
            public Path eval(FileStatus status) {
                Path out = status.getPath().makeQualified(localFs);
                LOG.debug("Loaded from DistributedCache: " + out);
                return out;
            }
        });
    }

    // If we are initializing a client-side instance to later serialize, the user may have
    // specified HDFS files, but also an intent to put the files in the DistributedCache. Set
    // this flag now, which will generate mDCachePrefix when addToConfiguration() is called
    // later.
    mUseDCache = conf.getBoolean(CONF_USE_DCACHE_KEY, USE_DCACHE_DEFAULT);
}

---

From source file:com.moz.fiji.schema.mapreduce.DistributedCacheJars.java

License:Apache License

/**
 * @param conf Configuration to get FileSystem from
 * @param jarDirectory The directory of jars to get.
 * @return A list of qualified paths to the jars in jarDirectory.
 * @throws IOException if there's a problem.
 *///from  ww  w .j a  v a  2s  .  c  om
public static List<String> getJarsFromDirectory(Configuration conf, File jarDirectory) throws IOException {
    if (!jarDirectory.isDirectory()) {
        throw new IOException("Attempted to add jars from non-directory: " + jarDirectory.getCanonicalPath());
    }
    List<String> allJars = new ArrayList<String>();
    FileSystem fileSystem = FileSystem.getLocal(conf);
    for (File jar : jarDirectory.listFiles()) {
        if (jar.exists() && !jar.isDirectory() && jar.getName().endsWith(".jar")) {
            Path jarPath = new Path(jar.getCanonicalPath());
            String qualifiedPath = jarPath.makeQualified(fileSystem).toString();
            allJars.add(qualifiedPath);
        }
    }
    return allJars;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »