Example usage for org.apache.hadoop.fs LocalFileSystem LocalFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs LocalFileSystem LocalFileSystem.

Prototype

public LocalFileSystem()

Source Link

Usage

From source file:org.apache.tajo.yarn.TajoPullServerService.java

License:Apache License

@Override
public void serviceInit(Configuration conf) throws Exception {
    tajoConf = new TajoConf(conf);

    manageOsCache = tajoConf.getBoolean(PullServerConstants.SHUFFLE_MANAGE_OS_CACHE,
            PullServerConstants.DEFAULT_SHUFFLE_MANAGE_OS_CACHE);

    readaheadLength = tajoConf.getInt(PullServerConstants.SHUFFLE_READAHEAD_BYTES,
            PullServerConstants.DEFAULT_SHUFFLE_READAHEAD_BYTES);

    int workerNum = tajoConf.getIntVar(ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM);

    ThreadFactory bossFactory = new ThreadFactoryBuilder().setNameFormat("TajoPullServerService Netty Boss #%d")
            .build();/*ww  w  . j  a  v a 2  s . c  om*/
    ThreadFactory workerFactory = new ThreadFactoryBuilder()
            .setNameFormat("TajoPullServerService Netty Worker #%d").build();
    selector = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory),
            Executors.newCachedThreadPool(workerFactory), workerNum);

    localFS = new LocalFileSystem();

    maxUrlLength = tajoConf.getIntVar(ConfVars.PULLSERVER_FETCH_URL_MAX_LENGTH);

    LOG.info("Tajo PullServer initialized: readaheadLength=" + readaheadLength);

    ServerBootstrap bootstrap = new ServerBootstrap(selector);
    try {
        channelInitializer = new HttpChannelInitializer(tajoConf);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    bootstrap.setPipelineFactory(channelInitializer);

    port = tajoConf.getIntVar(ConfVars.PULLSERVER_PORT);
    Channel ch = bootstrap.bind(new InetSocketAddress(port));

    accepted.add(ch);
    port = ((InetSocketAddress) ch.getLocalAddress()).getPort();
    tajoConf.set(ConfVars.PULLSERVER_PORT.varname, Integer.toString(port));
    LOG.info(getName() + " listening on port " + port);

    sslFileBufferSize = tajoConf.getInt(PullServerConstants.SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
            PullServerConstants.DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);

    int cacheSize = tajoConf.getIntVar(ConfVars.PULLSERVER_CACHE_SIZE);
    int cacheTimeout = tajoConf.getIntVar(ConfVars.PULLSERVER_CACHE_TIMEOUT);

    indexReaderCache = CacheBuilder.newBuilder().maximumSize(cacheSize)
            .expireAfterWrite(cacheTimeout, TimeUnit.MINUTES).removalListener(removalListener)
            .build(new CacheLoader<IndexCacheKey, BSTIndexReader>() {
                @Override
                public BSTIndexReader load(IndexCacheKey key) throws Exception {
                    return new BSTIndex(tajoConf).getIndexReader(new Path(key.getPath(), "index"));
                }
            });
    lowCacheHitCheckThreshold = (int) (cacheSize * 0.1f);

    super.serviceInit(tajoConf);
    LOG.info("TajoPullServerService started: port=" + port);
}

From source file:org.mule.modules.hdfs.automation.unit.HDFSConnectorTest.java

License:Open Source License

@Test
public void testDeleteFile() {
    try {/* w w  w .  j  a  v a  2s  .  c  om*/
        fileSystem = spy(new LocalFileSystem());
        doCallRealMethod().when(fileSystem).delete(any(Path.class), eq(false));
        connector.deleteFile("zoo");
    } catch (Exception e) {
        fail(ConnectorTestUtils.getStackTrace(e));
    }
}

From source file:org.mule.modules.hdfs.automation.unit.HDFSConnectorTest.java

License:Open Source License

@Test
public void testDeleteDirectory() {
    try {//  ww w  . j  a  va2  s  . c o  m
        fileSystem = spy(new LocalFileSystem());
        doCallRealMethod().when(fileSystem).delete(any(Path.class), eq(true));
        connector.deleteDirectory("zoo");
    } catch (Exception e) {
        fail(ConnectorTestUtils.getStackTrace(e));
    }
}

From source file:org.mule.modules.hdfs.automation.unit.HDFSConnectorTest.java

License:Open Source License

@Test
public void testMakeDirectories() {
    try {//from  w w  w . ja  v  a  2  s  . c o  m
        fileSystem = spy(new LocalFileSystem());
        doCallRealMethod().when(fileSystem).mkdirs(any(Path.class), any(FsPermission.class));
        connector.makeDirectories("foo", "755");
    } catch (Exception e) {
        fail(ConnectorTestUtils.getStackTrace(e));
    }
}