Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException 

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.util.SerializationUtilsTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    this.fs = FileSystem.getLocal(new Configuration());
    this.outputPath = new Path(SerializationUtilsTest.class.getSimpleName());
}

---

From source file:gobblin.yarn.GobblinHelixJobLauncherTest.java

License:Open Source License

@BeforeClass
public void setUp() throws Exception {
    this.closer.register(new TestingServer(TEST_ZK_PORT));

    URL url = GobblinHelixJobLauncherTest.class.getClassLoader()
            .getResource(GobblinHelixJobLauncherTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    Config config = ConfigFactory.parseURL(url).resolve();

    String zkConnectingString = config.getString(GobblinYarnConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    String helixClusterName = config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY);

    YarnHelixUtils.createGobblinYarnHelixCluster(zkConnectingString, helixClusterName);

    this.helixManager = HelixManagerFactory.getZKHelixManager(helixClusterName,
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectingString);
    this.helixManager.connect();

    Properties properties = ConfigUtils.configToProperties(config);

    this.localFs = FileSystem.getLocal(new Configuration());

    this.appWorkDir = new Path(GobblinHelixJobLauncherTest.class.getSimpleName());

    this.jobName = config.getString(ConfigurationKeys.JOB_NAME_KEY);

    this.jobOutputFile = new File(config.getString(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR),
            config.getString(ConfigurationKeys.WRITER_FILE_PATH) + File.separator
                    + config.getString(ConfigurationKeys.WRITER_FILE_NAME));

    // Prepare the source Json file
    File sourceJsonFile = new File(this.appWorkDir.toString(), TestHelper.TEST_JOB_NAME + ".json");
    TestHelper.createSourceJsonFile(sourceJsonFile);
    properties.setProperty(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, sourceJsonFile.getAbsolutePath());

    this.gobblinHelixJobLauncher = this.closer.register(new GobblinHelixJobLauncher(properties,
            this.helixManager, this.localFs, this.appWorkDir, ImmutableList.<Tag<?>>of()));

    this.gobblinWorkUnitRunner = new GobblinWorkUnitRunner(TestHelper.TEST_APPLICATION_NAME,
            TestHelper.TEST_HELIX_INSTANCE_NAME,
            ConverterUtils.toContainerId(TestHelper.TEST_PARTICIPANT_CONTAINER_ID), config,
            Optional.of(appWorkDir));

    this.fsDatasetStateStore = new FsDatasetStateStore(this.localFs,
            config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY));

    this.thread = new Thread(new Runnable() {
        @Override//from   w  ww  . ja v  a 2 s . c  om
        public void run() {
            gobblinWorkUnitRunner.start();
        }
    });
    this.thread.start();
}

---

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private void addLibJars(Path srcLibJarDir, Optional<Map<String, LocalResource>> resourceMap, Path destDir)
        throws IOException {
    FileSystem localFs = FileSystem.getLocal(this.yarnConfiguration);
    FileStatus[] libJarFiles = localFs.listStatus(srcLibJarDir);
    if (libJarFiles == null || libJarFiles.length == 0) {
        return;//from   w  w w.j  av a 2  s.  c o m
    }

    for (FileStatus libJarFile : libJarFiles) {
        Path destFilePath = new Path(destDir, libJarFile.getPath().getName());
        this.fs.copyFromLocalFile(libJarFile.getPath(), destFilePath);
        if (resourceMap.isPresent()) {
            YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE,
                    resourceMap.get());
        }
    }
}

---

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private void addJobConfPackage(String jobConfPackagePath, Path destDir, Map<String, LocalResource> resourceMap)
        throws IOException {
    Path srcFilePath = new Path(jobConfPackagePath);
    Path destFilePath = new Path(destDir,
            srcFilePath.getName() + GobblinClusterConfigurationKeys.TAR_GZ_FILE_SUFFIX);
    StreamUtils.tar(FileSystem.getLocal(this.yarnConfiguration), this.fs, srcFilePath, destFilePath);
    YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.ARCHIVE, resourceMap);
}

---

From source file:gobblin.yarn.GobblinYarnLogSource.java

License:Apache License

/**
 * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}.
 *
 * @param config the {@link Config} use to create the {@link LogCopier}
 * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in
 * @param destFs the destination {@link FileSystem}
 * @param appWorkDir the Gobblin Yarn application working directory on HDFS
 * @return a {@link LogCopier} instance//from  w ww .j ava  2s  .c o  m
 * @throws IOException if it fails on any IO operation
 */
protected LogCopier buildLogCopier(Config config, ContainerId containerId, FileSystem destFs, Path appWorkDir)
        throws IOException {
    LogCopier.Builder builder = LogCopier.newBuilder()
            .useSrcFileSystem(FileSystem.getLocal(new Configuration())).useDestFileSystem(destFs)
            .readFrom(getLocalLogDir()).writeTo(getHdfsLogDir(containerId, destFs, appWorkDir))
            .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR))
            .useLogFileNamePrefix(containerId.toString());
    if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) {
        builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE));
    }
    if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) {
        builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER));
    }
    return builder.build();
}

---

From source file:gobblin.yarn.YarnHelixUtilsTest.java

License:Open Source License

@BeforeClass
public void setUp() throws IOException {
    this.configuration = new Configuration();
    this.fileSystem = FileSystem.getLocal(this.configuration);
    this.tokenFilePath = new Path(YarnHelixUtilsTest.class.getSimpleName(), "token");
    this.token = new Token<>();
    this.token.setKind(new Text("test"));
    this.token.setService(new Text("test"));
}

---

From source file:gobblin.yarn.YarnSecurityManagerTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Use a random ZK port
    TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
    LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

    this.curatorFramework = this.closer.register(
            CuratorFrameworkFactory.newClient(testingZKServer.getConnectString(), new RetryOneTime(2000)));
    this.curatorFramework.start();

    URL url = YarnSecurityManagerTest.class.getClassLoader()
            .getResource(YarnSecurityManagerTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    Config config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string",
            ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve();

    String zkConnectingString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    String helixClusterName = config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);

    HelixUtils.createGobblinHelixCluster(zkConnectingString, helixClusterName);

    this.helixManager = HelixManagerFactory.getZKHelixManager(helixClusterName,
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.SPECTATOR, zkConnectingString);
    this.helixManager.connect();

    this.configuration = new Configuration();
    this.localFs = Mockito.spy(FileSystem.getLocal(this.configuration));

    this.token = new Token<>();
    this.token.setKind(new Text("test"));
    this.token.setService(new Text("test"));
    Mockito.<Token<?>>when(/* w  ww . ja v a 2s  .com*/
            this.localFs.getDelegationToken(UserGroupInformation.getLoginUser().getShortUserName()))
            .thenReturn(this.token);

    this.baseDir = new Path(YarnSecurityManagerTest.class.getSimpleName());
    this.tokenFilePath = new Path(this.baseDir, GobblinYarnConfigurationKeys.TOKEN_FILE_NAME);
    this.yarnAppSecurityManager = new YarnAppSecurityManager(config, this.helixManager, this.localFs,
            this.tokenFilePath);
    this.yarnContainerSecurityManager = new YarnContainerSecurityManager(config, this.localFs, new EventBus());
}

---

From source file:gr.ntua.h2rdf.loadTriples.TotalOrderPartitioner.java

License:Apache License

/**
 * Read in the partition file and build indexing data structures.
 * If the keytype is {@link org.apache.hadoop.io.BinaryComparable} and
 * <tt>total.order.partitioner.natural.order</tt> is not false, a trie
 * of the first <tt>total.order.partitioner.max.trie.depth</tt>(2) + 1 bytes
 * will be built. Otherwise, keys will be located using a binary search of
 * the partition keyset using the {@link org.apache.hadoop.io.RawComparator}
 * defined for this job. The input file must be sorted with the same
 * comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.
 */// w  ww.  j ava  2 s . c  om
@SuppressWarnings("unchecked") // keytype from conf not static
public void setConf(Configuration conf) {
    try {
        this.conf = conf;
        String parts = getPartitionFile(conf);
        final Path partFile = new Path(parts);
        final FileSystem fs = (DEFAULT_PATH.equals(parts)) ? FileSystem.getLocal(conf) // assume in DistributedCache
                : partFile.getFileSystem(conf);

        Job job = new Job(conf);
        Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
        K[] splitPoints = readPartitions(fs, partFile, keyClass, conf);
        if (splitPoints.length > job.getNumReduceTasks() - 1) {
            System.out.println(job.getNumReduceTasks());
            System.out.println(splitPoints.length);
            System.out.println("Wrong number of partitions in keyset:");
            throw new IOException("Wrong number of partitions in keyset:" + splitPoints.length);
        }
        RawComparator<K> comparator = (RawComparator<K>) job.getSortComparator();
        for (int i = 0; i < splitPoints.length - 1; ++i) {
            if (comparator.compare(splitPoints[i], splitPoints[i + 1]) >= 0) {
                throw new IOException("Split points are out of order");
            }
        }
        boolean natOrder = conf.getBoolean(NATURAL_ORDER, true);
        if (natOrder && BinaryComparable.class.isAssignableFrom(keyClass)) {
            partitions = buildTrie((BinaryComparable[]) splitPoints, 0, splitPoints.length, new byte[0],
                    // Now that blocks of identical splitless trie nodes are 
                    // represented reentrantly, and we develop a leaf for any trie
                    // node with only one split point, the only reason for a depth
                    // limit is to refute stack overflow or bloat in the pathological
                    // case where the split points are long and mostly look like bytes 
                    // iii...iixii...iii   .  Therefore, we make the default depth
                    // limit large but not huge.
                    conf.getInt(MAX_TRIE_DEPTH, 200));
        } else {
            partitions = new BinarySearchNode(splitPoints, comparator);
        }
    } catch (IOException e) {
        throw new IllegalArgumentException("Can't read partitions file", e);
    }
}

---

From source file:hrider.hbase.Connection.java

License:Apache License

/**
 * Saves a table locally to an HFile./* ww  w .j  a v a 2 s  .co m*/
 *
 * @param tableName The name of the table.
 * @param path      The path tot he file.
 * @throws IOException Error accessing hbase.
 */
public void saveTable(String tableName, String path) throws IOException {
    FileSystem fs = FileSystem.getLocal(this.getConfiguration());
    HTable table = this.factory.get(tableName);

    Configuration cacheConfig = new Configuration(this.getConfiguration());
    cacheConfig.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);

    StoreFile.Writer writer = new StoreFile.WriterBuilder(this.getConfiguration(), new CacheConfig(cacheConfig),
            fs, HColumnDescriptor.DEFAULT_BLOCKSIZE).withFilePath(new Path(path)).build();

    ResultScanner scanner = null;

    try {
        Scan scan = new Scan();
        scan.setCaching(GlobalConfig.instance().getBatchSizeForRead());

        scanner = table.getScanner(scan);

        boolean isValid;
        do {
            Result result = scanner.next();

            isValid = result != null;
            if (isValid) {
                for (KeyValue keyValue : result.list()) {
                    writer.append(keyValue);
                }

                for (HbaseActionListener listener : this.listeners) {
                    listener.saveOperation(tableName, path, result);
                }
            }
        } while (isValid);
    } finally {
        if (scanner != null) {
            scanner.close();
        }

        writer.close();
    }
}

---

From source file:hrider.hbase.Connection.java

License:Apache License

/**
 * Loads a locally saved HFile to an existing table.
 *
 * @param tableName The name of the table to load to.
 * @param path      The path to the HFile.
 * @throws IOException Error accessing hbase.
 *//*from w ww .j  a  v a2  s.c o m*/
public void loadTable(String tableName, String path) throws IOException, TableNotFoundException {
    FileSystem fs = FileSystem.getLocal(this.getConfiguration());
    HTable table = this.factory.get(tableName);

    HTableDescriptor td = this.hbaseAdmin.getTableDescriptor(Bytes.toBytes(tableName));

    Collection<ColumnFamily> families = new HashSet<ColumnFamily>();
    for (HColumnDescriptor column : td.getColumnFamilies()) {
        families.add(new ColumnFamily(column));
    }

    StoreFile.Reader reader = new StoreFile.Reader(fs, new Path(path),
            new CacheConfig(this.getConfiguration()));

    try {
        StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
        //SchemaMetrics.configureGlobally(this.getConfiguration());

        // move to the first row.
        scanner.seek(new KeyValue(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }));

        Collection<ColumnFamily> familiesToCreate = new HashSet<ColumnFamily>();

        Put put = null;
        List<Put> puts = new ArrayList<Put>();

        boolean isValid;
        int batchSize = GlobalConfig.instance().getBatchSizeForWrite();

        do {
            KeyValue kv = scanner.next();

            isValid = kv != null;
            if (isValid) {
                ColumnFamily columnFamily = new ColumnFamily(Bytes.toStringBinary(kv.getFamily()));
                if (!families.contains(columnFamily)) {
                    familiesToCreate.add(columnFamily);
                }

                if (put == null) {
                    put = new Put(kv.getRow());
                    puts.add(put);
                }

                if (!Arrays.equals(put.getRow(), kv.getRow())) {
                    for (HbaseActionListener listener : this.listeners) {
                        listener.loadOperation(tableName, path, put);
                    }

                    if (puts.size() == batchSize) {
                        if (!familiesToCreate.isEmpty()) {
                            createFamilies(tableName, toDescriptors(familiesToCreate));

                            families.addAll(familiesToCreate);
                            familiesToCreate.clear();
                        }

                        HTableUtil.bucketRsPut(table, puts);
                        puts.clear();
                    }

                    put = new Put(kv.getRow());
                    puts.add(put);
                }

                put.add(kv);
            }
        } while (isValid);

        // add the last put to the table.
        if (!puts.isEmpty()) {
            for (HbaseActionListener listener : this.listeners) {
                listener.loadOperation(tableName, path, put);
            }

            if (!familiesToCreate.isEmpty()) {
                createFamilies(tableName, toDescriptors(familiesToCreate));
            }

            HTableUtil.bucketRsPut(table, puts);
        }
    } finally {
        reader.close(false);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »