Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent() 

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:hsyndicate.hadoop.dfs.HSyndicateDFS.java

License:Apache License

@Override
public synchronized FSDataOutputStream create(Path file, FsPermission permission, boolean overwrite,
        int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
    SyndicateFSPath hpath = makeSyndicateFSPath(file);
    if (this.syndicateFS.exists(hpath)) {
        if (overwrite) {
            delete(file);/*from  www . ja  va  2  s .  c o m*/
        } else {
            throw new IOException("File already exists: " + file);
        }
    } else {
        Path parent = file.getParent();
        if (parent != null) {
            if (!mkdirs(parent)) {
                throw new IOException("Mkdirs failed to create " + parent.toString());
            }
        }
    }

    return new FSDataOutputStream(this.syndicateFS.getFileOutputStream(hpath), this.statistics);
}

---

From source file:IndexService.IndexServer.java

License:Open Source License

public boolean setindexstatus(String indexloc, int status) {
    if (!testmode) {
        Path indexpath = new Path(indexloc);
        try {/*w w  w . ja  v a  2  s  . c om*/
            String dbname = indexpath.getParent().getParent().getName();
            if (dbname.endsWith(".db")) {
                dbname = dbname.substring(0, dbname.lastIndexOf(".db"));
            }
            return db.setIndexStatus(dbname, indexpath.getParent().getName(), indexpath.getName(), status);
        } catch (HiveException e1) {
            e1.printStackTrace();
            return false;
        }
    } else {
        try {
            ArrayList<IndexItemStatus> statuss = new ArrayList<IndexItemStatus>();
            File file = new File("indexconf");
            boolean exist = false;
            if (file.exists()) {
                DataInputStream dis = new DataInputStream(new FileInputStream(file));
                int num = dis.readInt();
                for (int i = 0; i < num; i++) {
                    IndexItemStatus itemstatus = new IndexItemStatus();
                    itemstatus.read(dis);
                    if (itemstatus.indexlocation.equals(indexloc)) {
                        itemstatus.status = status;
                        exist = true;
                    }
                    statuss.add(itemstatus);
                }
                dis.close();

                DataOutputStream dos = new DataOutputStream(new FileOutputStream(file));
                dos.writeInt(statuss.size());
                for (IndexItemStatus indexItemStatus : statuss) {
                    indexItemStatus.write(dos);
                }
                dos.close();
            }
            if (exist)
                return true;
            return false;
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return false;
    }
}

---

From source file:IndexService.IndexServer_Old.java

License:Open Source License

public static void main(String[] args) throws IOException, HiveException {
    IndexServer_Old server = new IndexServer_Old();
    server.start();//  ww  w  .  j a v  a2 s .co m
    BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
    String str;
    while (true) {
        try {
            str = br.readLine();
            String[] strs = str.split(" ");
            if (strs[0].equals("setindexstatus")) {
                Path indexpath = new Path(strs[1]);
                server.db.setIndexStatus(indexpath.getParent().getParent().getName(),
                        indexpath.getParent().getName(), indexpath.getName(), Integer.valueOf(strs[2]));
            } else if (str.equals("exit")) {
                server.close();
                System.exit(0);
                break;
            }
        } catch (NumberFormatException e) {
            e.printStackTrace();
        } catch (HiveException e) {
            e.printStackTrace();
        }
    }
}

---

From source file:info.halo9pan.word2vec.hadoop.Main.java

License:Open Source License

/**
 * @param args//from   www . ja  v a  2s.co  m
 */
public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    conf.set("hadoop.tmp.dir", (new Path("temp")).toUri().toString());
    GenericOptionsParser optionParser = new GenericOptionsParser(conf, args);
    String[] remainingArgs = optionParser.getRemainingArgs();
    if (!(remainingArgs.length != 2 || remainingArgs.length != 4)) {
        System.err.println("Usage: wordcount <in> <out> [-skip skipPatternFile]");
        System.exit(2);
    }
    Path input = new Path(remainingArgs[0]);
    String inputName = input.getName();
    Path countOutput = new Path(input.getParent(), inputName + "_count");
    Job countJob = Job.getInstance(conf, "Word Count");
    countJob.setJarByClass(Main.class);
    countJob.setMapperClass(ReadWordsMapper.class);
    countJob.setCombinerClass(ReadWordsReducer.class);
    countJob.setReducerClass(ReadWordsReducer.class);
    countJob.setOutputKeyClass(Text.class);
    countJob.setOutputValueClass(IntWritable.class);

    FileInputFormat.setInputPaths(countJob, input);
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(countOutput))
        fs.delete(countOutput, true);
    FileOutputFormat.setOutputPath(countJob, countOutput);
    countJob.waitForCompletion(true);

    System.exit(countJob.waitForCompletion(true) ? 0 : 1);
}

---

From source file:io.druid.indexer.updater.HadoopConverterJob.java

License:Apache License

public List<DataSegment> run() throws IOException {
    final JobConf jobConf = new JobConf();
    jobConf.setKeepFailedTaskFiles(false);
    for (Map.Entry<String, String> entry : converterConfig.getHadoopProperties().entrySet()) {
        jobConf.set(entry.getKey(), entry.getValue(), "converterConfig.getHadoopProperties()");
    }/* ww  w .ja va2  s .com*/
    final List<DataSegment> segments = converterConfig.getSegments();
    if (segments.isEmpty()) {
        throw new IAE("No segments found for datasource [%s]", converterConfig.getDataSource());
    }
    converterConfigIntoConfiguration(converterConfig, segments, jobConf);

    jobConf.setNumReduceTasks(0);// Map only. Number of map tasks determined by input format
    jobConf.setWorkingDirectory(new Path(converterConfig.getDistributedSuccessCache()));

    setJobName(jobConf, segments);

    if (converterConfig.getJobPriority() != null) {
        jobConf.setJobPriority(JobPriority.valueOf(converterConfig.getJobPriority()));
    }

    final Job job = Job.getInstance(jobConf);

    job.setInputFormatClass(ConfigInputFormat.class);
    job.setMapperClass(ConvertingMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setMapSpeculativeExecution(false);
    job.setOutputFormatClass(ConvertingOutputFormat.class);

    JobHelper.setupClasspath(JobHelper.distributedClassPath(jobConf.getWorkingDirectory()),
            JobHelper.distributedClassPath(getJobClassPathDir(job.getJobName(), jobConf.getWorkingDirectory())),
            job);

    Throwable throwable = null;
    try {
        job.submit();
        log.info("Job %s submitted, status available at %s", job.getJobName(), job.getTrackingURL());
        final boolean success = job.waitForCompletion(true);
        if (!success) {
            final TaskReport[] reports = job.getTaskReports(TaskType.MAP);
            if (reports != null) {
                for (final TaskReport report : reports) {
                    log.error("Error in task [%s] : %s", report.getTaskId(),
                            Arrays.toString(report.getDiagnostics()));
                }
            }
            return null;
        }
        try {
            loadedBytes = job.getCounters().findCounter(COUNTER_GROUP, COUNTER_LOADED).getValue();
            writtenBytes = job.getCounters().findCounter(COUNTER_GROUP, COUNTER_WRITTEN).getValue();
        } catch (IOException ex) {
            log.error(ex, "Could not fetch counters");
        }
        final JobID jobID = job.getJobID();

        final Path jobDir = getJobPath(jobID, job.getWorkingDirectory());
        final FileSystem fs = jobDir.getFileSystem(job.getConfiguration());
        final RemoteIterator<LocatedFileStatus> it = fs.listFiles(jobDir, true);
        final List<Path> goodPaths = new ArrayList<>();
        while (it.hasNext()) {
            final LocatedFileStatus locatedFileStatus = it.next();
            if (locatedFileStatus.isFile()) {
                final Path myPath = locatedFileStatus.getPath();
                if (ConvertingOutputFormat.DATA_SUCCESS_KEY.equals(myPath.getName())) {
                    goodPaths.add(new Path(myPath.getParent(), ConvertingOutputFormat.DATA_FILE_KEY));
                }
            }
        }
        if (goodPaths.isEmpty()) {
            log.warn("No good data found at [%s]", jobDir);
            return null;
        }
        final List<DataSegment> returnList = ImmutableList
                .copyOf(Lists.transform(goodPaths, new Function<Path, DataSegment>() {
                    @Nullable
                    @Override
                    public DataSegment apply(final Path input) {
                        try {
                            if (!fs.exists(input)) {
                                throw new ISE("Somehow [%s] was found but [%s] is missing at [%s]",
                                        ConvertingOutputFormat.DATA_SUCCESS_KEY,
                                        ConvertingOutputFormat.DATA_FILE_KEY, jobDir);
                            }
                        } catch (final IOException e) {
                            throw Throwables.propagate(e);
                        }
                        try (final InputStream stream = fs.open(input)) {
                            return HadoopDruidConverterConfig.jsonMapper.readValue(stream, DataSegment.class);
                        } catch (final IOException e) {
                            throw Throwables.propagate(e);
                        }
                    }
                }));
        if (returnList.size() == segments.size()) {
            return returnList;
        } else {
            throw new ISE(
                    "Tasks reported success but result length did not match! Expected %d found %d at path [%s]",
                    segments.size(), returnList.size(), jobDir);
        }
    } catch (InterruptedException | ClassNotFoundException e) {
        RuntimeException exception = Throwables.propagate(e);
        throwable = exception;
        throw exception;
    } catch (Throwable t) {
        throwable = t;
        throw t;
    } finally {
        try {
            cleanup(job);
        } catch (IOException e) {
            if (throwable != null) {
                throwable.addSuppressed(e);
            } else {
                log.error(e, "Could not clean up job [%s]", job.getJobID());
            }
        }
    }
}

---

From source file:io.druid.segment.loading.HdfsDataSegmentFinderTest.java

License:Apache License

private String getDescriptorPath(DataSegment segment) {
    final Path indexzip = new Path(String.valueOf(segment.getLoadSpec().get("path")));
    return indexzip.getParent().toString() + "/" + DESCRIPTOR_JSON;
}

---

From source file:io.druid.segment.loading.HdfsDataSegmentFinderTest.java

License:Apache License

private String getDescriptorPathWithPartitionNum(DataSegment segment, int partitionNum) {
    final Path indexzip = new Path(String.valueOf(segment.getLoadSpec().get("path")));
    return indexzip.getParent().toString() + "/" + partitionNum + "_" + DESCRIPTOR_JSON;
}

---

From source file:io.druid.segment.loading.HdfsDataSegmentPusher.java

License:Open Source License

@Override
public DataSegment push(File inDir, DataSegment segment) throws IOException {
    final String storageDir = DataSegmentPusherUtil.getHdfsStorageDir(segment);
    Path outFile = new Path(String.format("%s/%s/index.zip", config.getStorageDirectory(), storageDir));
    FileSystem fs = outFile.getFileSystem(hadoopConfig);

    fs.mkdirs(outFile.getParent());
    log.info("Compressing files from[%s] to [%s]", inDir, outFile);
    FSDataOutputStream out = null;/* w  w w . j a  v a2 s  .  co m*/
    long size;
    try {
        out = fs.create(outFile);

        size = CompressionUtils.zip(inDir, out);

        out.close();
    } finally {
        Closeables.closeQuietly(out);
    }

    return createDescriptorFile(segment.withLoadSpec(makeLoadSpec(outFile)).withSize(size)
            .withBinaryVersion(IndexIO.CURRENT_VERSION_ID), outFile.getParent(), fs);
}

---

From source file:io.druid.storage.hdfs.HdfsDataSegmentFinder.java

License:Apache License

@Override
public Set<DataSegment> findSegments(String workingDirPathStr, boolean updateDescriptor)
        throws SegmentLoadingException {
    final Set<DataSegment> segments = Sets.newHashSet();
    final Path workingDirPath = new Path(workingDirPathStr);
    FileSystem fs;// w w w  .j  a  v  a  2  s  . c om
    try {
        fs = workingDirPath.getFileSystem(config);

        log.info(fs.getScheme());
        log.info("FileSystem URI:" + fs.getUri().toString());

        if (!fs.exists(workingDirPath)) {
            throw new SegmentLoadingException("Working directory [%s] doesn't exist.", workingDirPath);
        }

        if (!fs.isDirectory(workingDirPath)) {
            throw new SegmentLoadingException("Working directory [%s] is not a directory!?", workingDirPath);
        }

        final RemoteIterator<LocatedFileStatus> it = fs.listFiles(workingDirPath, true);
        while (it.hasNext()) {
            final LocatedFileStatus locatedFileStatus = it.next();
            final Path path = locatedFileStatus.getPath();
            if (path.getName().endsWith("descriptor.json")) {
                final Path indexZip;
                final String descriptorParts[] = path.getName().split("_");
                if (descriptorParts.length == 2 && descriptorParts[1].equals("descriptor.json")
                        && org.apache.commons.lang.StringUtils.isNumeric(descriptorParts[0])) {
                    indexZip = new Path(path.getParent(),
                            StringUtils.format("%s_index.zip", descriptorParts[0]));
                } else {
                    indexZip = new Path(path.getParent(), "index.zip");
                }
                if (fs.exists(indexZip)) {
                    final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class);
                    log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip);

                    final Map<String, Object> loadSpec = dataSegment.getLoadSpec();
                    final String pathWithoutScheme = indexZip.toUri().getPath();

                    if (!loadSpec.get("type").equals(HdfsStorageDruidModule.SCHEME)
                            || !loadSpec.get("path").equals(pathWithoutScheme)) {
                        loadSpec.put("type", HdfsStorageDruidModule.SCHEME);
                        loadSpec.put("path", pathWithoutScheme);
                        if (updateDescriptor) {
                            log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", path,
                                    pathWithoutScheme);
                            mapper.writeValue(fs.create(path, true), dataSegment);
                        }
                    }
                    segments.add(dataSegment);
                } else {
                    throw new SegmentLoadingException(
                            "index.zip didn't exist at [%s] while descripter.json exists!?", indexZip);
                }
            }
        }
    } catch (IOException e) {
        throw new SegmentLoadingException(e, "Problems interacting with filesystem[%s].", workingDirPath);
    }

    return segments;
}

---

From source file:io.druid.storage.hdfs.HdfsDataSegmentKiller.java

License:Apache License

@Override
public void kill(DataSegment segment) throws SegmentLoadingException {
    final Path path = getPath(segment);
    final FileSystem fs = checkPathAndGetFilesystem(path);
    try {//from  w  w w . ja v  a 2s  . c  o  m
        if (path.getName().endsWith(".zip")) {

            // path format -- > .../dataSource/interval/version/partitionNum/xxx.zip
            Path partitionNumDir = path.getParent();
            if (!fs.delete(partitionNumDir, true)) {
                throw new SegmentLoadingException("Unable to kill segment, failed to delete dir [%s]",
                        partitionNumDir.toString());
            }

            //try to delete other directories if possible
            Path versionDir = partitionNumDir.getParent();
            if (safeNonRecursiveDelete(fs, versionDir)) {
                Path intervalDir = versionDir.getParent();
                if (safeNonRecursiveDelete(fs, intervalDir)) {
                    Path dataSourceDir = intervalDir.getParent();
                    safeNonRecursiveDelete(fs, dataSourceDir);
                }
            }
        } else {
            throw new SegmentLoadingException("Unknown file type[%s]", path);
        }
    } catch (IOException e) {
        throw new SegmentLoadingException(e, "Unable to kill segment");
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »