Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException 

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.quantcast.qfs.hadoop.QFSEmulationImpl.java

License:Apache License

public QFSEmulationImpl(Configuration conf) throws IOException {
    localFS = FileSystem.getLocal(conf);
    umask = FsPermission.getUMask(conf).toShort();
}

---

From source file:com.quantcast.qfs.hadoop.QuantcastFileSystem.java

License:Apache License

public void initialize(URI uri, Configuration conf) throws IOException {
    super.initialize(uri, conf);
    setConf(conf);/*  ww w.  ja  v a2s . c o  m*/

    try {
        if (qfsImpl == null) {
            if (uri.getHost() == null) {
                qfsImpl = createIFSImpl(conf.get("fs.qfs.metaServerHost", ""),
                        conf.getInt("fs.qfs.metaServerPort", -1), statistics, conf);
            } else {
                qfsImpl = createIFSImpl(uri.getHost(), uri.getPort(), statistics, conf);
            }
        }

        this.localFs = FileSystem.getLocal(conf);
        this.uri = URI
                .create(uri.getScheme() + "://" + (uri.getAuthority() == null ? "/" : uri.getAuthority()));
        this.workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(uri, null);
        this.qfsImpl.setUMask(FsPermission.getUMask(conf).toShort());
    } catch (Exception e) {
        throw new IOException("Unable to initialize QFS using uri " + uri);
    }
}

---

From source file:com.ricemap.spateDB.core.SpatialSite.java

License:Apache License

public static CellInfo[] getCells(JobConf job) throws IOException {
    CellInfo[] cells = null;/* w w w .  ja v  a2 s. c om*/
    String cells_file = job.get(OUTPUT_CELLS);
    if (cells_file != null) {
        Path[] cacheFiles = DistributedCache.getLocalCacheFiles(job);
        for (Path cacheFile : cacheFiles) {
            if (cacheFile.getName().contains(cells_file)) {
                FSDataInputStream in = FileSystem.getLocal(job).open(cacheFile);

                int cellCount = in.readInt();
                cells = new CellInfo[cellCount];
                for (int i = 0; i < cellCount; i++) {
                    cells[i] = new CellInfo();
                    cells[i].readFields(in);
                }

                in.close();
            }
        }
    }
    return cells;
}

---

From source file:com.sensei.indexing.hadoop.reduce.ShardWriter.java

License:Apache License

/**
 * Constructor//from w  w w .  j  a  va2  s .com
 * @param fs
 * @param shard
 * @param tempDir
 * @param iconf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration iconf) throws IOException {
    logger.info("Construct a shard writer");

    this.iconf = iconf;
    this.fs = fs;
    localFs = FileSystem.getLocal(iconf);
    perm = new Path(shard.getDirectory());
    temp = new Path(tempDir);

    long initGeneration = shard.getGeneration();

    if (localFs.exists(temp)) {
        File tempFile = new File(temp.getName());
        if (tempFile.exists())
            SenseiReducer.deleteDir(tempFile);
    }

    if (!fs.exists(perm)) {
        assert (initGeneration < 0);
        fs.mkdirs(perm);
    } else {
        moveToTrash(iconf, perm);
        fs.mkdirs(perm);
        //      restoreGeneration(fs, perm, initGeneration);
    }
    //    dir =  //new FileSystemDirectory(fs, perm, false, iconf.getConfiguration());
    //        new MixedDirectory(fs, perm, localFs, fs.startLocalOutput(perm, temp),
    //            iconf);

    // analyzer is null because we only use addIndexes, not addDocument
    //    writer =
    //        new IndexWriter(dir, null, 
    //              initGeneration < 0 ? new KeepOnlyLastCommitDeletionPolicy() : new MixedDeletionPolicy(), 
    //                    MaxFieldLength.UNLIMITED);

    //    writer =  new IndexWriter(dir, null, new KeepOnlyLastCommitDeletionPolicy(), MaxFieldLength.UNLIMITED);
    writer = new IndexWriter(FSDirectory.open(new File(tempDir)), null, new KeepOnlyLastCommitDeletionPolicy(),
            MaxFieldLength.UNLIMITED);
    setParameters(iconf);
    //    dir = null;
    //    writer = null;

}

---

From source file:com.singular.utils.FileUtils.java

License:Open Source License

public static Path transferToSystem(Configuration configuration, Path localPath, Path remoteDir,
        String remoteFileName) {/*from www  .j av a 2  s . c om*/
    Path finalRemotePath = null;
    InputStream in = null;
    OutputStream out = null;

    System.out.println(
            "Transferring local " + localPath + " ,remoteStaging " + remoteDir + " ,fileNam=" + remoteFileName);

    try {
        FileSystem localFs = FileSystem.getLocal(configuration);
        FileSystem remoteFs = FileSystem.get(configuration);

        in = localFs.open(localPath);
        finalRemotePath = new Path(remoteDir, remoteFileName);
        out = remoteFs.create(finalRemotePath);
        copyStream(in, out);
    } catch (Exception e) {
        throw new IllegalArgumentException("Exception while transferring file.", e);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        if (out != null) {
            try {
                out.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    return finalRemotePath;
}

---

From source file:com.skp.experiment.fpm.pfpgrowth.PFPGrowth.java

License:Apache License

/**
 * Generates the fList from the serialized string representation
 * /*from   www  .j  a  v a2 s. c  o  m*/
 * @return Deserialized Feature Frequency List
 */
public static List<Pair<String, Long>> readFList(Configuration conf) throws IOException {
    List<Pair<String, Long>> list = new ArrayList<Pair<String, Long>>();
    Path[] files = DistributedCache.getLocalCacheFiles(conf);
    if (files == null) {
        throw new IOException("Cannot read Frequency list from Distributed Cache");
    }
    if (files.length != 1) {
        throw new IOException("Cannot read Frequency list from Distributed Cache (" + files.length + ")");
    }
    FileSystem fs = FileSystem.getLocal(conf);
    Path fListLocalPath = fs.makeQualified(files[0]);
    // Fallback if we are running locally.
    if (!fs.exists(fListLocalPath)) {
        URI[] filesURIs = DistributedCache.getCacheFiles(conf);
        if (filesURIs == null) {
            throw new IOException("Cannot read Frequency list from Distributed Cache");
        }
        if (filesURIs.length != 1) {
            throw new IOException("Cannot read Frequency list from Distributed Cache (" + files.length + ")");
        }
        fListLocalPath = new Path(filesURIs[0].getPath());
    }
    for (Pair<Text, LongWritable> record : new SequenceFileIterable<Text, LongWritable>(fListLocalPath, true,
            conf)) {
        list.add(new Pair<String, Long>(record.getFirst().toString(), record.getSecond().get()));
    }
    return list;
}

---

From source file:com.splicemachine.hbase.MockSnapshot.java

License:Apache License

public static void createFile(Path p) throws IOException {
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///tmp");
    FileSystem fs = FileSystem.getLocal(conf);

    FSDataOutputStream dos = fs.create(p, true);
    dos.write(0);/* ww w  .j a va 2s . c  om*/
    dos.flush();
    dos.close();
}

---

From source file:com.splicemachine.mrio.api.SpliceTableMapReduceUtil.java

License:Apache License

/**
 * Add the jars containing the given classes to the job's configuration
 * such that JobClient will ship them to the cluster and add them to
 * the DistributedCache.//from   ww w .  j a va 2  s  . com
 */
public static void addDependencyJars(Configuration conf, Class... classes) throws IOException {

    FileSystem localFs = FileSystem.getLocal(conf);

    Set<String> jars = new HashSet<String>();

    // Add jars that are already in the tmpjars variable
    jars.addAll(conf.getStringCollection("tmpjars"));

    // Add jars containing the specified classes
    for (Class clazz : classes) {
        if (clazz == null)
            continue;

        String pathStr = findOrCreateJar(clazz);
        if (pathStr == null) {
            LOG.warn("Could not find jar for class " + clazz + " in order to ship it to the cluster.");
            continue;
        }
        Path path = new Path(pathStr);
        if (!localFs.exists(path)) {
            LOG.warn("Could not validate jar file " + path + " for class " + clazz);
            continue;
        }
        jars.add(path.makeQualified(localFs).toString());
    }
    if (jars.isEmpty())
        return;

    conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0])));
}

---

From source file:com.splout.db.benchmark.BenchmarkStoreTool.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    JCommander jComm = new JCommander(this);
    jComm.setProgramName("Benchmark-Store Tool");
    try {//from  ww w.  ja va2s  .com
        jComm.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jComm.usage();
        return -1;
    } catch (Throwable t) {
        t.printStackTrace();
        jComm.usage();
        return -1;
    }

    // Create some input files that will represent the partitions to generate
    Path out = new Path(output);
    FileSystem outFs = out.getFileSystem(getConf());
    HadoopUtils.deleteIfExists(outFs, out);

    Integer min, max, eachPartition;
    int maxKeyDigits;

    try {
        String[] minMax = keySpace.split(":");
        min = Integer.parseInt(minMax[0]);
        max = Integer.parseInt(minMax[1]);
        maxKeyDigits = max.toString().length();

        eachPartition = (max - min) / nPartitions;
    } catch (Exception e) {
        throw new IllegalArgumentException(
                "Key range format is not valid. It must be minKey:maxKey where both minKey and maxKey are integers.");
    }

    FileSystem inFs = FileSystem.get(getConf());
    Path input = new Path("benchmark-store-tool-" + System.currentTimeMillis());
    HadoopUtils.deleteIfExists(inFs, input);
    inFs.mkdirs(input);

    List<PartitionEntry> partitionEntries = new ArrayList<PartitionEntry>();

    // Create as many input files as partitions
    // Each input file will have as value the range that each Mapper will write
    String paddingExp = "%0" + (padding != null ? padding : maxKeyDigits) + "d";
    for (int i = 0; i < nPartitions; i++) {
        int thisMin = (i * eachPartition);
        int thisMax = (i + 1) * eachPartition;
        HadoopUtils.stringToFile(inFs, new Path(input, i + ".txt"), i + "\t" + thisMin + ":" + thisMax);
        PartitionEntry entry = new PartitionEntry();
        entry.setMin(String.format(paddingExp, thisMin));
        entry.setMax(String.format(paddingExp, thisMax));
        entry.setShard(i);
        partitionEntries.add(entry);
    }

    partitionEntries.get(0).setMin(null);
    partitionEntries.get(partitionEntries.size() - 1).setMax(null);

    PartitionMap partitionMap = new PartitionMap(partitionEntries);
    HadoopUtils.stringToFile(outFs, new Path(out, "partition-map"), JSONSerDe.ser(partitionMap));

    List<Field> fields = new ArrayList<Field>();
    fields.add(Field.create(SploutSQLOutputFormat.PARTITION_TUPLE_FIELD, Type.INT));
    fields.addAll(Fields.parse("key:int, value:string"));
    final Schema schema = new Schema(tablename, fields);

    byte[] valueArray = new byte[valueSize];
    for (int i = 0; i < valueSize; i++) {
        valueArray[i] = 'A';
    }
    final String theValue = new String(valueArray);

    if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) {
        File nativeLibs = new File("native");
        if (nativeLibs.exists()) {
            SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf);
        }
    }

    MapOnlyJobBuilder job = new MapOnlyJobBuilder(conf);
    TableSpec tableSpec = new TableSpec(schema, schema.getFields().get(1));

    job.setOutput(new Path(out, "store"),
            new SploutSQLProxyOutputFormat(new SQLite4JavaOutputFormat(1000000, tableSpec)), ITuple.class,
            NullWritable.class);
    job.addInput(input, new HadoopInputFormat(TextInputFormat.class),
            new MapOnlyMapper<LongWritable, Text, ITuple, NullWritable>() {

                ITuple metaTuple = new Tuple(schema);

                protected void map(LongWritable key, Text value, Context context)
                        throws IOException, InterruptedException {

                    String[] partitionRange = value.toString().split("\t");
                    Integer partition = Integer.parseInt(partitionRange[0]);
                    metaTuple.set(SploutSQLOutputFormat.PARTITION_TUPLE_FIELD, partition);
                    String[] minMax = partitionRange[1].split(":");
                    Integer min = Integer.parseInt(minMax[0]);
                    Integer max = Integer.parseInt(minMax[1]);
                    for (int i = min; i < max; i++) {
                        metaTuple.set("key", i);
                        metaTuple.set("value", theValue);
                        context.write(metaTuple, NullWritable.get());
                    }
                }
            });

    job.createJob().waitForCompletion(true);

    HadoopUtils.deleteIfExists(inFs, input);
    return 0;
}

---

From source file:com.splout.db.dnode.Fetcher.java

License:Open Source License

private File hdfsFetch(Path fromPath, Reporter reporter) throws IOException, InterruptedException {
    UUID uniqueId = UUID.randomUUID();
    File toFile = new File(tempDir, uniqueId.toString() + "/" + fromPath.getName());
    File toDir = new File(toFile.getParent());
    if (toDir.exists()) {
        FileUtils.deleteDirectory(toDir);
    }//from   ww w . jav a  2  s.  c o  m
    toDir.mkdirs();
    Path toPath = new Path(toFile.getCanonicalPath());

    FileSystem fS = fromPath.getFileSystem(hadoopConf);
    FileSystem tofS = FileSystem.getLocal(hadoopConf);

    Throttler throttler = new Throttler((double) bytesPerSecThrottle);
    try {
        for (FileStatus fStatus : fS.globStatus(fromPath)) {
            log.info("Copying " + fStatus.getPath() + " to " + toPath);
            long bytesSoFar = 0;

            FSDataInputStream iS = fS.open(fStatus.getPath());
            FSDataOutputStream oS = tofS.create(toPath);

            byte[] buffer = new byte[downloadBufferSize];

            int nRead;
            while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
                // Needed to being able to be interrupted at any moment.
                if (Thread.interrupted()) {
                    iS.close();
                    oS.close();
                    cleanDirNoExceptions(toDir);
                    throw new InterruptedException();
                }
                bytesSoFar += nRead;
                oS.write(buffer, 0, nRead);
                throttler.incrementAndThrottle(nRead);
                if (bytesSoFar >= bytesToReportProgress) {
                    reporter.progress(bytesSoFar);
                    bytesSoFar = 0l;
                }
            }

            if (reporter != null) {
                reporter.progress(bytesSoFar);
            }

            oS.close();
            iS.close();
        }

        return toDir;
    } catch (ClosedByInterruptException e) {
        // This can be thrown by the method read.
        cleanDirNoExceptions(toDir);
        throw new InterruptedIOException();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »