Example usage for org.apache.hadoop.fs Path SEPARATOR

List of usage examples for org.apache.hadoop.fs Path SEPARATOR

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:gobblin.yarn.GobblinHelixJobLauncher.java

License:Open Source License

public GobblinHelixJobLauncher(Properties jobProps, HelixManager helixManager, FileSystem fs, Path appWorkDir,
        List<? extends Tag<?>> metadataTags) throws Exception {
    super(jobProps, metadataTags);

    this.helixManager = helixManager;
    this.helixTaskDriver = new TaskDriver(this.helixManager);

    this.fs = fs;
    this.appWorkDir = appWorkDir;
    this.inputWorkUnitDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.INPUT_WORK_UNIT_DIR_NAME);
    this.outputTaskStateDir = new Path(this.appWorkDir, GobblinYarnConfigurationKeys.OUTPUT_TASK_STATE_DIR_NAME
            + Path.SEPARATOR + this.jobContext.getJobId());

    this.helixQueueName = this.jobContext.getJobName();
    this.jobResourceName = TaskUtil.getNamespacedJobName(this.helixQueueName, this.jobContext.getJobId());

    this.jobContext.getJobState().setJobLauncherType(LauncherTypeEnum.YARN);

    this.stateSerDeRunnerThreads = Integer
            .parseInt(jobProps.getProperty(ParallelRunner.PARALLEL_RUNNER_THREADS_KEY,
                    Integer.toString(ParallelRunner.DEFAULT_PARALLEL_RUNNER_THREADS)));

    this.taskStateCollectorService = new TaskStateCollectorService(jobProps, this.jobContext.getJobState(),
            this.eventBus, this.fs, outputTaskStateDir);
}

From source file:gobblin.yarn.GobblinHelixTaskTest.java

License:Open Source License

private void prepareWorkUnit(WorkUnit workUnit) {
    workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, TestHelper.TEST_TASK_ID);
    workUnit.setProp(ConfigurationKeys.SOURCE_CLASS_KEY, SimpleJsonSource.class.getName());
    workUnit.setProp(ConfigurationKeys.CONVERTER_CLASSES_KEY, SimpleJsonConverter.class.getName());
    workUnit.setProp(ConfigurationKeys.WRITER_OUTPUT_FORMAT_KEY, WriterOutputFormat.AVRO.toString());
    workUnit.setProp(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY,
            Destination.DestinationType.HDFS.toString());
    workUnit.setProp(ConfigurationKeys.WRITER_STAGING_DIR,
            this.appWorkDir.toString() + Path.SEPARATOR + "staging");
    workUnit.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, this.taskOutputDir.toString());
    workUnit.setProp(ConfigurationKeys.WRITER_FILE_NAME, TestHelper.WRITER_FILE_NAME);
    workUnit.setProp(ConfigurationKeys.WRITER_FILE_PATH, TestHelper.REL_WRITER_FILE_PATH);
    workUnit.setProp(ConfigurationKeys.WRITER_BUILDER_CLASS, AvroDataWriterBuilder.class.getName());
    workUnit.setProp(ConfigurationKeys.SOURCE_SCHEMA, TestHelper.SOURCE_SCHEMA);
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

/**
 * Launch a new Gobblin instance on Yarn.
 *
 * @throws IOException if there's something wrong launching the application
 * @throws YarnException if there's something wrong launching the application
 *///from w w w. ja  v a  2  s  .  co  m
public void launch() throws IOException, YarnException {
    this.eventBus.register(this);

    String clusterName = this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);
    HelixUtils.createGobblinHelixCluster(
            this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY), clusterName);
    LOGGER.info("Created Helix cluster " + clusterName);

    connectHelixManager();

    startYarnClient();

    this.applicationId = getApplicationId();

    this.applicationStatusMonitor.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            try {
                eventBus.post(new ApplicationReportArrivalEvent(
                        yarnClient.getApplicationReport(applicationId.get())));
            } catch (YarnException | IOException e) {
                LOGGER.error(
                        "Failed to get application report for Gobblin Yarn application " + applicationId.get(),
                        e);
                eventBus.post(new GetApplicationReportFailureEvent(e));
            }
        }
    }, 0, this.appReportIntervalMinutes, TimeUnit.MINUTES);

    List<Service> services = Lists.newArrayList();
    if (this.config.hasPath(GobblinYarnConfigurationKeys.KEYTAB_FILE_PATH)) {
        LOGGER.info("Adding YarnAppSecurityManager since login is keytab based");
        services.add(buildYarnAppSecurityManager());
    }
    if (!this.config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_DISABLE_DRIVER_COPY)
            || !this.config.getBoolean(GobblinYarnConfigurationKeys.LOG_COPIER_DISABLE_DRIVER_COPY)) {
        services.add(buildLogCopier(this.config,
                new Path(this.sinkLogRootDir,
                        this.applicationName + Path.SEPARATOR + this.applicationId.get().toString()),
                GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName,
                        this.applicationId.get().toString())));
    }
    if (config.getBoolean(ConfigurationKeys.JOB_EXECINFO_SERVER_ENABLED_KEY)) {
        LOGGER.info("Starting the job execution info server since it is enabled");
        Properties properties = ConfigUtils.configToProperties(config);
        JobExecutionInfoServer executionInfoServer = new JobExecutionInfoServer(properties);
        services.add(executionInfoServer);
        if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) {
            LOGGER.info("Starting the admin UI server since it is enabled");
            services.add(new AdminWebServer(properties, executionInfoServer.getAdvertisedServerUri()));
        }
    } else if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) {
        LOGGER.warn("NOT starting the admin UI because the job execution info server is NOT enabled");
    }

    this.serviceManager = Optional.of(new ServiceManager(services));
    // Start all the services running in the ApplicationMaster
    this.serviceManager.get().startAsync();
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private YarnAppSecurityManager buildYarnAppSecurityManager() throws IOException {
    Path tokenFilePath = new Path(this.fs.getHomeDirectory(),
            this.applicationName + Path.SEPARATOR + GobblinYarnConfigurationKeys.TOKEN_FILE_NAME);
    return new YarnAppSecurityManager(this.config, this.helixManager, this.fs, tokenFilePath);
}

From source file:gobblin.yarn.YarnContainerSecurityManager.java

License:Apache License

public YarnContainerSecurityManager(Config config, FileSystem fs, EventBus eventBus) {
    this.fs = fs;
    this.tokenFilePath = new Path(this.fs.getHomeDirectory(),
            config.getString(GobblinYarnConfigurationKeys.APPLICATION_NAME_KEY) + Path.SEPARATOR
                    + GobblinYarnConfigurationKeys.TOKEN_FILE_NAME);
    this.eventBus = eventBus;
}

From source file:hdfs.FileUtil.java

License:Apache License

private static void checkDependencies(FileSystem srcFS, Path src, FileSystem dstFS, Path dst)
        throws IOException {
    if (srcFS == dstFS) {
        String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
        String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
        if (dstq.startsWith(srcq)) {
            if (srcq.length() == dstq.length()) {
                throw new IOException("Cannot copy " + src + " to itself.");
            } else {
                throw new IOException("Cannot copy " + src + " to its subdirectory " + dst);
            }//from  w w  w  .  j a  va 2  s.c o  m
        }
    }
}

From source file:internal.coprocessor.NonAggregatingRegionObserver.java

License:Apache License

/**
 * Builds the regionIndex by going through all the store files for each region and creating a reader for it that can
 * later be used to read the bloom filters, on demand.
 *///w  w  w  .  j a  v  a 2 s  .  c o  m
@VisibleForTesting
public static NavigableMap<ByteBuffer, ListMultimap<ByteBuffer, BloomFilter>> buildIndex(String tableName,
        Configuration conf, FileSystem fileSystem, Path tableBasePath) throws IOException {
    ImmutableSortedMap.Builder<ByteBuffer, ListMultimap<ByteBuffer, BloomFilter>> indexBuilder = ImmutableSortedMap
            .naturalOrder();
    try {
        HTable table = new HTable(conf, tableName);
        NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
        Collection<HColumnDescriptor> families = table.getTableDescriptor().getFamilies();
        table.close();
        LOG.info("Building RegionIndex [Table: " + tableName + " Base Path: " + tableBasePath.toString() + "]");
        for (HRegionInfo region : regions.keySet()) {
            ImmutableListMultimap.Builder<ByteBuffer, BloomFilter> familiesAndStoreFiles = ImmutableListMultimap
                    .builder();
            for (HColumnDescriptor family : families) {
                Path path = new Path(tableBasePath,
                        region.getEncodedName() + Path.SEPARATOR + family.getNameAsString());
                FileStatus[] storeFilesStatuses = fileSystem.listStatus(path);
                for (FileStatus status : storeFilesStatuses) {
                    LOG.info("Processing Store File: " + status.getPath().toString() + " Column Family: "
                            + family.getNameAsString() + " for Region: " + region.getRegionNameAsString());
                    // maybe increased the cache for this reader since it's be only reading blooms?
                    HFile.Reader reader = HFile.createReader(fileSystem, status.getPath(),
                            new CacheConfig(conf));
                    familiesAndStoreFiles.put(ByteBuffer.wrap(family.getName()),
                            BloomFilterFactory.createFromMeta(reader.getGeneralBloomFilterMetadata(), reader));
                }
            }
            indexBuilder.put(ByteBuffer.wrap(region.getStartKey()), familiesAndStoreFiles.build());
        }
        return indexBuilder.build();
    } catch (Exception e) {
        LOG.error("Could not load regionIndex, BloomFilterSemiJoinFilter will not work.", e);
    }
    return null;
}

From source file:io.dataapps.chlorine.hadoop.DeepScanPipeline.java

License:Apache License

public void run() {
    try {/* w w  w  . j a va 2  s. c o m*/
        final Path fsScanPath = new Path(scanPath);
        final Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        if (findersFilePath != null) {
            fs.copyFromLocalFile(false, true, new Path(findersFilePath), new Path("chlorine_finders.xml"));
        }
        Job job = HDFSScanMR.makeJob(conf, fsScanPath, new Path(jobOutputDir), matchPath, scanSince,
                findersFilePath, queue, maskPath);
        boolean bResult = runJobToCompletion(job);
        if (bResult) {
            LOG.info("Total bytes scanned = "
                    + job.getCounters().findCounter("Feature", "TotalSize").getValue());
            LOG.info("Total records scanned = " + job.getCounters()
                    .findCounter("org.apache.hadoop.mapreduce.TaskCounter", "MAP_INPUT_RECORDS").getValue());
            LOG.info("Total Matched records = "
                    + job.getCounters().findCounter("Feature", "MatchedRecords").getValue());
            LOG.info("Total matches = " + job.getCounters().findCounter("Feature", "TotalMatches").getValue());
            FinderEngine engine = new FinderEngine();
            for (Finder finder : engine.getFinders()) {
                long l = job.getCounters().findCounter("Feature", finder.getName()).getValue();
                if (l > 0) {
                    LOG.info(finder.getName() + " = "
                            + job.getCounters().findCounter("Feature", "TotalMatches").getValue());
                }
            }
            if (matchPath != null) {
                String tempMatchesPath = jobOutputDir + Path.SEPARATOR + "_temp";
                String matchOutputPath = matchPath + Path.SEPARATOR + "scan_result_" + scanPath.hashCode() + "_"
                        + scanSince;
                FileUtil.copyMerge(fs, new Path(tempMatchesPath), fs, new Path(matchOutputPath), true, conf,
                        null);
                LOG.info("The matches detected are stored in " + matchOutputPath);
            }

            if (maskPath != null) {
                LOG.info("The matches in the input are masked and a copy is kept under " + maskPath);

            }
        }

    } catch (IOException e) {
        LOG.error(e);
    }
}

From source file:io.druid.storage.hdfs.tasklog.HdfsTaskLogs.java

License:Apache License

private static String mergePaths(String path1, String path2) {
    return path1 + (path1.endsWith(Path.SEPARATOR) ? "" : Path.SEPARATOR) + path2;
}

From source file:io.hops.common.INodeUtil.java

License:Apache License

public static String constructPath(byte[][] components, int start, int end) {
    StringBuilder buf = new StringBuilder();
    for (int i = start; i < end; i++) {
        buf.append(DFSUtil.bytes2String(components[i]));
        if (i < end - 1) {
            buf.append(Path.SEPARATOR);
        }/*from   w ww .j  av a  2s . co  m*/
    }
    return buf.toString();
}