Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent() 

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:org.apache.accumulo.server.master.tableOps.BulkImport.java

License:Apache License

@Override
public Repo<Master> call(long tid, Master master) throws Exception {
    // This needs to execute after the arbiter is stopped

    VolumeManager fs = master.getFileSystem();

    if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
        return new CleanUpBulkImport(tableId, source, bulk, error);

    HashMap<String, String> failures = new HashMap<String, String>();
    HashMap<String, String> loadedFailures = new HashMap<String, String>();

    FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT));
    BufferedReader in = new BufferedReader(new InputStreamReader(failFile));
    try {//w  w w.j a  v  a 2s.c  o  m
        String line = null;
        while ((line = in.readLine()) != null) {
            Path path = new Path(line);
            if (!fs.exists(new Path(error, path.getName())))
                failures.put("/" + path.getParent().getName() + "/" + path.getName(), line);
        }
    } finally {
        failFile.close();
    }

    /*
     * I thought I could move files that have no file references in the table. However its possible a clone references a file. Therefore only move files that
     * have no loaded markers.
     */

    // determine which failed files were loaded
    Connector conn = master.getConnector();
    Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
    mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);

    for (Entry<Key, Value> entry : mscanner) {
        if (Long.parseLong(entry.getValue().toString()) == tid) {
            String loadedFile = entry.getKey().getColumnQualifier().toString();
            String absPath = failures.remove(loadedFile);
            if (absPath != null) {
                loadedFailures.put(loadedFile, absPath);
            }
        }
    }

    // move failed files that were not loaded
    for (String failure : failures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());
        fs.rename(orig, dest);
        log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
    }

    if (loadedFailures.size() > 0) {
        DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/"
                + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZBULK_FAILED_COPYQ);

        HashSet<String> workIds = new HashSet<String>();

        for (String failure : loadedFailures.values()) {
            Path orig = new Path(failure);
            Path dest = new Path(error, orig.getName());

            if (fs.exists(dest))
                continue;

            bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes());
            workIds.add(orig.getName());
            log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
        }

        bifCopyQueue.waitUntilDone(workIds);
    }

    fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
    return new CleanUpBulkImport(tableId, source, bulk, error);
}

---

From source file:org.apache.accumulo.server.tabletserver.BulkFailedCopyProcessor.java

License:Apache License

@Override
public void process(String workID, byte[] data) {

    String paths[] = new String(data).split(",");

    Path orig = new Path(paths[0]);
    Path dest = new Path(paths[1]);
    Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");

    try {//from   ww  w .java 2s .  c o  m
        FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.FileUtil
                .getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration()));

        FileUtil.copy(fs, orig, fs, tmp, false, true, CachedConfiguration.getInstance());
        fs.rename(tmp, dest);
        log.debug("copied " + orig + " to " + dest);
    } catch (IOException ex) {
        try {
            FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.FileUtil.getFileSystem(
                    CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration()));

            fs.create(dest).close();
            log.warn(" marked " + dest + " failed", ex);
        } catch (IOException e) {
            log.error("Unable to create failure flag file " + dest, e);
        }
    }

}

---

From source file:org.apache.accumulo.server.tabletserver.ScanRunState.java

License:Apache License

public void recover(VolumeManager fs, Tablet tablet, List<LogEntry> logEntries, Set<String> tabletFiles,
        MutationReceiver mutationReceiver) throws IOException {
    List<Path> recoveryLogs = new ArrayList<Path>();
    List<LogEntry> sorted = new ArrayList<LogEntry>(logEntries);
    Collections.sort(sorted, new Comparator<LogEntry>() {
        @Override//from  ww  w .jav  a 2s . c  o m
        public int compare(LogEntry e1, LogEntry e2) {
            return (int) (e1.timestamp - e2.timestamp);
        }
    });
    for (LogEntry entry : sorted) {
        Path recovery = null;
        for (String log : entry.logSet) {
            String[] parts = log.split("/", 2); // "host:port/filename"
            Path finished = new Path(fs.getFullPath(FileType.RECOVERY, parts[parts.length - 1]), "finished");
            TabletServer.log.info("Looking for " + finished);
            if (fs.exists(finished)) {
                recovery = finished.getParent();
                break;
            }
        }
        if (recovery == null)
            throw new IOException(
                    "Unable to find recovery files for extent " + tablet.getExtent() + " logEntry: " + entry);
        recoveryLogs.add(recovery);
    }
    logger.recover(fs, tablet, recoveryLogs, tabletFiles, mutationReceiver);
}

---

From source file:org.apache.accumulo.test.functional.BulkFailureIT.java

License:Apache License

private static void newLoad(long txid, ClientContext context, KeyExtent extent, Path path, long size,
        boolean expectFailure) throws Exception {

    TabletClientService.Iface client = getClient(context, extent);
    try {//from ww w . j a v  a 2s  .com

        Map<String, MapFileInfo> val = ImmutableMap.of(path.getName(), new MapFileInfo(size));
        Map<KeyExtent, Map<String, MapFileInfo>> files = ImmutableMap.of(extent, val);

        client.loadFiles(TraceUtil.traceInfo(), context.rpcCreds(), txid, path.getParent().toString(),
                Translator.translate(files, Translators.KET), false);

        if (!expectFailure) {
            while (!getLoaded(context, extent).contains(path)) {
                Thread.sleep(100);
            }
        }

    } finally {
        ThriftUtil.returnClient((TServiceClient) client);
    }
}

---

From source file:org.apache.accumulo.test.functional.CleanTmpIT.java

License:Apache License

@Test
public void test() throws Exception {
    Connector c = getConnector();/*from  w  ww  . ja  v  a 2s  .  co  m*/
    // make a table
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    // write to it
    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("row");
    m.put("cf", "cq", "value");
    bw.addMutation(m);
    bw.flush();

    // Compact memory to make a file
    c.tableOperations().compact(tableName, null, null, true, true);

    // Make sure that we'll have a WAL
    m = new Mutation("row2");
    m.put("cf", "cq", "value");
    bw.addMutation(m);
    bw.close();

    // create a fake _tmp file in its directory
    String id = c.tableOperations().tableIdMap().get(tableName);
    Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
    s.setRange(Range.prefix(id));
    s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
    Entry<Key, Value> entry = Iterables.getOnlyElement(s);
    Path file = new Path(entry.getKey().getColumnQualifier().toString());

    FileSystem fs = getCluster().getFileSystem();
    assertTrue("Could not find file: " + file, fs.exists(file));
    Path tabletDir = file.getParent();
    assertNotNull("Tablet dir should not be null", tabletDir);
    Path tmp = new Path(tabletDir, "junk.rf_tmp");
    // Make the file
    fs.create(tmp).close();
    log.info("Created tmp file {}", tmp.toString());
    getCluster().stop();
    getCluster().start();

    Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
    assertEquals(2, Iterators.size(scanner.iterator()));
    // If we performed log recovery, we should have cleaned up any stray files
    assertFalse("File still exists: " + tmp, fs.exists(tmp));
}

---

From source file:org.apache.accumulo.test.functional.CloneTestIT.java

License:Apache License

private void checkMetadata(String table, Connector conn) throws Exception {
    Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);

    s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
    MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
    String tableId = conn.tableOperations().tableIdMap().get(table);

    Assert.assertNotNull("Could not get table id for " + table, tableId);

    s.setRange(Range.prefix(tableId));/*  w  ww  . j  a  v a2 s.c  o m*/

    Key k;
    Text cf = new Text(), cq = new Text();
    int itemsInspected = 0;
    for (Entry<Key, Value> entry : s) {
        itemsInspected++;
        k = entry.getKey();
        k.getColumnFamily(cf);
        k.getColumnQualifier(cq);

        if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) {
            Path p = new Path(cq.toString());
            FileSystem fs = cluster.getFileSystem();
            Assert.assertTrue("File does not exist: " + p, fs.exists(p));
        } else if (cf
                .equals(MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily())) {
            Assert.assertEquals("Saw unexpected cq",
                    MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), cq);
            Path tabletDir = new Path(entry.getValue().toString());
            Path tableDir = tabletDir.getParent();
            Path tablesDir = tableDir.getParent();

            Assert.assertEquals(ServerConstants.TABLE_DIR, tablesDir.getName());
        } else {
            Assert.fail("Got unexpected key-value: " + entry);
            throw new RuntimeException();
        }
    }

    Assert.assertTrue("Expected to find metadata entries", itemsInspected > 0);
}

---

From source file:org.apache.accumulo.test.VolumeIT.java

License:Apache License

@Test
public void testRelativePaths() throws Exception {

    List<String> expected = new ArrayList<>();

    Connector connector = getConnector();
    String tableName = getUniqueNames(1)[0];
    connector.tableOperations().create(tableName, new NewTableConfiguration().withoutDefaultIterators());

    String tableId = connector.tableOperations().tableIdMap().get(tableName);

    SortedSet<Text> partitions = new TreeSet<>();
    // with some splits
    for (String s : "c,g,k,p,s,v".split(","))
        partitions.add(new Text(s));

    connector.tableOperations().addSplits(tableName, partitions);

    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());

    // create two files in each tablet

    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
    for (String s : rows) {
        Mutation m = new Mutation(s);
        m.put("cf1", "cq1", "1");
        bw.addMutation(m);/*from   w w w . j a v  a  2s.c  o  m*/
        expected.add(s + ":cf1:cq1:1");
    }

    bw.flush();
    connector.tableOperations().flush(tableName, null, null, true);

    for (String s : rows) {
        Mutation m = new Mutation(s);
        m.put("cf1", "cq1", "2");
        bw.addMutation(m);
        expected.add(s + ":cf1:cq1:2");
    }

    bw.close();
    connector.tableOperations().flush(tableName, null, null, true);

    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));

    connector.tableOperations().offline(tableName, true);

    connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);

    Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
    metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
    metaScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());

    BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());

    for (Entry<Key, Value> entry : metaScanner) {
        String cq = entry.getKey().getColumnQualifier().toString();
        if (cq.startsWith(v1.toString())) {
            Path path = new Path(cq);
            String relPath = "/" + path.getParent().getName() + "/" + path.getName();
            Mutation fileMut = new Mutation(entry.getKey().getRow());
            fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
            fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
            mbw.addMutation(fileMut);
        }
    }

    mbw.close();

    connector.tableOperations().online(tableName, true);

    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));

    connector.tableOperations().compact(tableName, null, null, true, true);

    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));

    for (Entry<Key, Value> entry : metaScanner) {
        String cq = entry.getKey().getColumnQualifier().toString();
        Path path = new Path(cq);
        Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
    }

}

---

From source file:org.apache.accumulo.tserver.BulkFailedCopyProcessor.java

License:Apache License

@Override
public void process(String workID, byte[] data) {

    String paths[] = new String(data, UTF_8).split(",");

    Path orig = new Path(paths[0]);
    Path dest = new Path(paths[1]);
    Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");

    try {/*  w w w  . j  a  v a 2s  .  co  m*/
        VolumeManager vm = VolumeManagerImpl.get(SiteConfiguration.getInstance());
        FileSystem origFs = vm.getVolumeByPath(orig).getFileSystem();
        FileSystem destFs = vm.getVolumeByPath(dest).getFileSystem();

        FileUtil.copy(origFs, orig, destFs, tmp, false, true, CachedConfiguration.getInstance());
        destFs.rename(tmp, dest);
        log.debug("copied " + orig + " to " + dest);
    } catch (IOException ex) {
        try {
            VolumeManager vm = VolumeManagerImpl.get(SiteConfiguration.getInstance());
            FileSystem destFs = vm.getVolumeByPath(dest).getFileSystem();
            destFs.create(dest).close();
            log.warn(" marked " + dest + " failed", ex);
        } catch (IOException e) {
            log.error("Unable to create failure flag file " + dest, e);
        }
    }

}

---

From source file:org.apache.accumulo.tserver.DirectoryDecommissioner.java

License:Apache License

public static Path checkTabletDirectory(TabletServer tserver, VolumeManager vm, KeyExtent extent, Path dir)
        throws IOException {
    if (isActiveVolume(dir))
        return dir;

    if (!dir.getParent().getParent().getName().equals(ServerConstants.TABLE_DIR)) {
        throw new IllegalArgumentException("Unexpected table dir " + dir);
    }//from w ww .  jav a  2s  . c o  m

    Path newDir = new Path(
            vm.choose(ServerConstants.getTablesDirs()) + "/" + dir.getParent().getName() + "/" + dir.getName());

    log.info("Updating directory for " + extent + " from " + dir + " to " + newDir);
    if (extent.isRootTablet()) {
        // the root tablet is special case, its files need to be copied if its dir is changed

        // this code needs to be idempotent

        FileSystem fs1 = vm.getFileSystemByPath(dir);
        FileSystem fs2 = vm.getFileSystemByPath(newDir);

        if (!same(fs1, dir, fs2, newDir)) {
            if (fs2.exists(newDir)) {
                Path newDirBackup = getBackupName(fs2, newDir);
                // never delete anything because were dealing with the root tablet
                // one reason this dir may exist is because this method failed previously
                log.info("renaming " + newDir + " to " + newDirBackup);
                if (!fs2.rename(newDir, newDirBackup)) {
                    throw new IOException("Failed to rename " + newDir + " to " + newDirBackup);
                }
            }

            // do a lot of logging since this is the root tablet
            log.info("copying " + dir + " to " + newDir);
            if (!FileUtil.copy(fs1, dir, fs2, newDir, false, CachedConfiguration.getInstance())) {
                throw new IOException("Failed to copy " + dir + " to " + newDir);
            }

            // only set the new location in zookeeper after a successful copy
            log.info("setting root tablet location to " + newDir);
            MetadataTableUtil.setRootTabletDir(newDir.toString());

            // rename the old dir to avoid confusion when someone looks at filesystem... its ok if we fail here and this does not happen because the location in
            // zookeeper is the authority
            Path dirBackup = getBackupName(fs1, dir);
            log.info("renaming " + dir + " to " + dirBackup);
            fs1.rename(dir, dirBackup);
        } else {
            log.info("setting root tablet location to " + newDir);
            MetadataTableUtil.setRootTabletDir(newDir.toString());
        }

        return dir;
    } else {
        MetadataTableUtil.updateTabletDir(extent, newDir.toString(), SystemCredentials.get(),
                tserver.getLock());
        return newDir;
    }
}

---

From source file:org.apache.accumulo.tserver.DirectoryDecommissioner.java

License:Apache License

private static Path getBackupName(FileSystem fs, Path path) {
    SecureRandom rand = new SecureRandom();
    return new Path(path.getParent(),
            path.getName() + "_" + System.currentTimeMillis() + "_" + Math.abs(rand.nextInt()) + ".bak");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »