Example usage for org.apache.hadoop.fs CreateFlag CREATE

List of usage examples for org.apache.hadoop.fs CreateFlag CREATE

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. CreateFlag
  6. CREATE

Introduction

In this page you can find the example usage for org.apache.hadoop.fs CreateFlag CREATE.

Prototype

CreateFlag CREATE

To view the source code for org.apache.hadoop.fs CreateFlag CREATE.

Click Source Link

Document

Create a file.

Usage

From source file:com.conductor.hadoop.WritableValueInputFormat.java

License:Apache License

/**
 * Writes the provided {@code values} to an input file to be read by the {@link Job}, and sets up all additional
 * necessary configuration./*from  ww  w  . ja  va2s  .c  om*/
 * 
 * @param values
 *            the values to be read by the job.
 * @param clazz
 *            the type of the values.
 * @param inputsPerSplit
 *            how man inputs each split gets
 * @param job
 *            the job to configure
 * @param <V>
 *            the type of the {@code values}
 * @throws IOException
 */
public static <V extends Writable> void setupInput(final List<V> values, Class<V> clazz,
        final int inputsPerSplit, final Job job) throws IOException {
    final Path inputPath = new Path("job_input_" + System.currentTimeMillis() + UUID.randomUUID().toString());
    final Writer writer = SequenceFile.createWriter(FileContext.getFileContext(job.getConfiguration()),
            job.getConfiguration(), inputPath, NullWritable.class, clazz, CompressionType.NONE, CODEC,
            new Metadata(), EnumSet.of(CreateFlag.CREATE), DUMMY_VAR_ARGS);
    doSetupInput(values, clazz, inputsPerSplit, job, inputPath, writer);
}

From source file:com.datatorrent.common.util.AsyncFSStorageAgent.java

License:Apache License

public void copyToHDFS(final int operatorId, final long windowId) throws IOException {
    if (this.localBasePath == null) {
        throw new AssertionError("save() was not called before copyToHDFS");
    }/*from w  ww. j  a  va2  s  .  com*/
    String operatorIdStr = String.valueOf(operatorId);
    File directory = new File(localBasePath, operatorIdStr);
    String window = Long.toHexString(windowId);
    Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
    File srcFile = new File(directory, String.valueOf(windowId));
    FSDataOutputStream stream = null;
    boolean stateSaved = false;
    try {
        // Create the temporary file with OverWrite option to avoid dangling lease issue and avoid exception if file already exists
        stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                Options.CreateOpts.CreateParent.createParent());
        InputStream in = null;
        try {
            in = new FileInputStream(srcFile);
            IOUtils.copyBytes(in, stream, conf, false);
        } finally {
            IOUtils.closeStream(in);
        }
        stateSaved = true;
    } catch (Throwable t) {
        logger.debug("while saving {} {}", operatorId, window, t);
        stateSaved = false;
        throw Throwables.propagate(t);
    } finally {
        try {
            if (stream != null) {
                stream.close();
            }
        } catch (IOException ie) {
            stateSaved = false;
            throw new RuntimeException(ie);
        } finally {
            if (stateSaved) {
                fileContext.rename(lPath,
                        new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
                        Options.Rename.OVERWRITE);
            }
            FileUtil.fullyDelete(srcFile);
        }
    }
}

From source file:com.datatorrent.common.util.FSStorageAgent.java

License:Apache License

@SuppressWarnings("ThrowFromFinallyBlock")
@Override//from  www  . j av a 2 s .c om
public void save(Object object, int operatorId, long windowId) throws IOException {
    String operatorIdStr = String.valueOf(operatorId);
    Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
    String window = Long.toHexString(windowId);
    boolean stateSaved = false;
    FSDataOutputStream stream = null;
    try {
        stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                Options.CreateOpts.CreateParent.createParent());
        store(stream, object);
        stateSaved = true;
    } catch (Throwable t) {
        logger.debug("while saving {} {}", operatorId, window, t);
        stateSaved = false;
        DTThrowable.rethrow(t);
    } finally {
        try {
            if (stream != null) {
                stream.close();
            }
        } catch (IOException ie) {
            stateSaved = false;
            throw new RuntimeException(ie);
        } finally {
            if (stateSaved) {
                logger.debug("Saving {}: {}", operatorId, window);
                fileContext.rename(lPath,
                        new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
                        Options.Rename.OVERWRITE);
            }
        }
    }
}

From source file:com.datatorrent.stram.FSRecoveryHandler.java

License:Apache License

@Override
public Object restore() throws IOException {
    FileContext fc = FileContext.getFileContext(fs.getUri());

    // recover from wherever it was left
    if (fc.util().exists(snapshotBackupPath)) {
        LOG.warn("Incomplete checkpoint, reverting to {}", snapshotBackupPath);
        fc.rename(snapshotBackupPath, snapshotPath, Rename.OVERWRITE);

        // combine logs (w/o append, create new file)
        Path tmpLogPath = new Path(basedir, "log.combined");
        FSDataOutputStream fsOut = fc.create(tmpLogPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE));
        try {//  ww w .j  a v  a 2s . com
            FSDataInputStream fsIn = fc.open(logBackupPath);
            try {
                IOUtils.copy(fsIn, fsOut);
            } finally {
                fsIn.close();
            }

            fsIn = fc.open(logPath);
            try {
                IOUtils.copy(fsIn, fsOut);
            } finally {
                fsIn.close();
            }
        } finally {
            fsOut.close();
        }

        fc.rename(tmpLogPath, logPath, Rename.OVERWRITE);
        fc.delete(logBackupPath, false);
    } else {
        // we have log backup, but no checkpoint backup
        // failure between log rotation and writing checkpoint
        if (fc.util().exists(logBackupPath)) {
            LOG.warn("Found {}, did checkpointing fail?", logBackupPath);
            fc.rename(logBackupPath, logPath, Rename.OVERWRITE);
        }
    }

    if (!fc.util().exists(snapshotPath)) {
        LOG.debug("No existing checkpoint.");
        return null;
    }

    LOG.debug("Reading checkpoint {}", snapshotPath);
    InputStream is = fc.open(snapshotPath);
    // indeterministic class loading behavior
    // http://stackoverflow.com/questions/9110677/readresolve-not-working-an-instance-of-guavas-serializedform-appears
    final ClassLoader loader = Thread.currentThread().getContextClassLoader();
    ObjectInputStream ois = new ObjectInputStream(is) {
        @Override
        protected Class<?> resolveClass(ObjectStreamClass objectStreamClass)
                throws IOException, ClassNotFoundException {
            return Class.forName(objectStreamClass.getName(), true, loader);
        }
    };
    //ObjectInputStream ois = new ObjectInputStream(is);
    try {
        return ois.readObject();
    } catch (ClassNotFoundException cnfe) {
        throw new IOException("Failed to read checkpointed state", cnfe);
    } finally {
        ois.close();
    }
}

From source file:com.ikanow.aleph2.aleph2_rest_utils.FileUtils.java

License:Apache License

public static void writeFile(final FileContext fileContext, final InputStream input, final String path)
        throws AccessControlException, FileAlreadyExistsException, FileNotFoundException,
        ParentNotDirectoryException, UnsupportedFileSystemException, IOException {
    final Path p = new Path(path);
    try (FSDataOutputStream outer = fileContext.create(p, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
            org.apache.hadoop.fs.Options.CreateOpts.createParent())) {
        IOUtils.copyLarge(input, outer, new byte[DEFAULT_BUFFER_SIZE]);
    }//from   www  .  java  2 s  .  com
}

From source file:com.ikanow.aleph2.analytics.r.services.BeJobLauncher.java

License:Apache License

/** Cache the system and user classpaths
 * @param job//from  w w w. ja v  a 2 s.c o  m
 * @param context
 * @throws IOException 
 * @throws ExecutionException 
 * @throws InterruptedException 
 * @throws IllegalArgumentException 
 */
protected static void cacheJars(final Job job, final DataBucketBean bucket, final IAnalyticsContext context)
        throws IllegalArgumentException, InterruptedException, ExecutionException, IOException {
    final FileContext fc = context.getServiceContext().getStorageService()
            .getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get();
    final String rootPath = context.getServiceContext().getStorageService().getRootPath();

    // Aleph2 libraries: need to cache them
    context.getAnalyticsContextLibraries(Optional.empty()).stream().map(f -> new File(f))
            .map(f -> Tuples._2T(f, new Path(rootPath + "/" + f.getName()))).map(Lambdas.wrap_u(f_p -> {
                final FileStatus fs = Lambdas.get(() -> {
                    //TODO (ALEPH-12): need to clear out the cache intermittently
                    try {
                        return fc.getFileStatus(f_p._2());
                    } catch (Exception e) {
                        return null;
                    }
                });
                if (null == fs) { //cache doesn't exist
                    // Local version
                    try (FSDataOutputStream outer = fc.create(f_p._2(), EnumSet.of(CreateFlag.CREATE), // ie should fail if the destination file already exists 
                            org.apache.hadoop.fs.Options.CreateOpts.createParent())) {
                        Files.copy(f_p._1(), outer.getWrappedStream());
                    } catch (FileAlreadyExistsException e) {//(carry on - the file is versioned so it can't be out of date)
                    }
                }
                return f_p._2();
            })).forEach(Lambdas.wrap_consumer_u(path -> job.addFileToClassPath(path)));
    ;

    // User libraries: this is slightly easier since one of the 2 keys
    // is the HDFS path (the other is the _id)
    context.getAnalyticsLibraries(Optional.of(bucket), bucket.analytic_thread().jobs()).get().entrySet()
            .stream().map(kv -> kv.getKey()).filter(path -> path.startsWith(rootPath))
            .forEach(Lambdas.wrap_consumer_u(path -> job.addFileToClassPath(new Path(path))));
    ;
}

From source file:com.ikanow.aleph2.analytics.spark.utils.SparkTechnologyUtils.java

License:Apache License

/** Cache the system and user classpaths and return HDFS paths
 * @param bucket/*from   w  w w . java 2  s.  com*/
 * @param main_jar_path - my JAR path
 * @param context
 * @throws IOException 
 * @throws ExecutionException 
 * @throws InterruptedException 
 * @throws IllegalArgumentException 
 */
public static List<String> getCachedJarList(final DataBucketBean bucket, final String main_jar_path,
        final IAnalyticsContext context)
        throws IllegalArgumentException, InterruptedException, ExecutionException, IOException {
    final FileContext fc = context.getServiceContext().getStorageService()
            .getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get();
    final String root_path = context.getServiceContext().getStorageService().getRootPath();
    final String tmp_dir = System.getProperty("java.io.tmpdir");

    // Aleph2 libraries: need to cache them
    final Stream<String> context_stream = context.getAnalyticsContextLibraries(Optional.empty()).stream()
            .filter(jar -> !jar.equals(main_jar_path)) // (this is the service case, eg "/opt/aleph2-home/lib/aleph2_spark_analytic_services.jar")
            .map(Lambdas.wrap_u(f_str -> {

                final Tuple3<File, Path, FileStatus> f_p_fs = f_str.contains("core_distributed_services")
                        || f_str.contains("data_model") ? removeSparkConflictsAndCache(f_str, root_path, fc)
                                : checkCache(f_str, root_path, fc);

                if (null == f_p_fs._3()) { //cache doesn't exist
                    // Local version
                    try (FSDataOutputStream outer = fc.create(f_p_fs._2(), EnumSet.of(CreateFlag.CREATE), // ie should fail if the destination file already exists 
                            org.apache.hadoop.fs.Options.CreateOpts.createParent())) {
                        Files.copy(f_p_fs._1(), outer.getWrappedStream());
                    } catch (FileAlreadyExistsException e) {//(carry on - the file is versioned so it can't be out of date)
                    }
                    if (f_p_fs._1().getPath().startsWith(tmp_dir)) { // (delete tmp files)
                        f_p_fs._1().delete();
                    }
                }

                return f_p_fs._2();
            })).map(p -> transformFromPath(p.toString()));

    // User libraries: this is slightly easier since one of the 2 keys
    // is the HDFS path (the other is the _id)
    final Stream<String> lib_stream = context
            .getAnalyticsLibraries(Optional.of(bucket), bucket.analytic_thread().jobs()).get().entrySet()
            .stream().map(kv -> kv.getKey()).filter(jar -> !jar.equals(main_jar_path)) // (this is the uploaded case, eg "/app/aleph2/library/blah.jar")
            .filter(path -> path.startsWith(root_path)).map(s -> transformFromPath(s));

    return Stream.concat(context_stream, lib_stream).collect(Collectors.toList());
}

From source file:com.ikanow.aleph2.core.shared.utils.DirUtils.java

License:Apache License

/** Creates a text file  in the storage service 
 * @param fileContext// w  w  w  .jav  a 2s .  co m
 * @param fileNameString
 * @param sb
 */
public static void createUTF8File(FileContext fileContext, String fileNameString, StringBuffer sb) {
    if (fileContext != null && fileNameString != null) {
        try {
            Path f = new Path(fileNameString);
            FSDataOutputStream out = fileContext.create(f, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE));
            //Read from input stream and write to output stream until EOF.
            Charset charset = StandardCharsets.UTF_8;
            CharsetEncoder encoder = charset.newEncoder();

            // No allocation performed, just wraps the StringBuilder.
            CharBuffer buffer = CharBuffer.wrap(sb.toString());

            byte[] bytes = encoder.encode(buffer).array();
            out.write(bytes);
            out.close();

        } catch (Exception e) {
            logger.error("createFolderStructure Caught Exception", e);
        }
    }

}

From source file:com.ikanow.aleph2.core.shared.utils.TestJarCacheUtils.java

License:Apache License

@Before
public void setUpDependencies()
        throws AccessControlException, FileAlreadyExistsException, FileNotFoundException,
        ParentNotDirectoryException, UnsupportedFileSystemException, IllegalArgumentException, IOException {

    final String temp_dir = System.getProperty("java.io.tmpdir") + File.separator;

    _globals = new GlobalPropertiesBean(temp_dir, temp_dir, temp_dir, temp_dir);

    _mock_hdfs = new MockStorageService(_globals);

    // Create a pretend "remote" file

    _test_file_path = temp_dir + "/test.jar";
    final Path test_file_path = new Path(_test_file_path);

    final FileContext fs = _mock_hdfs.getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get();
    fs.create(test_file_path, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE));

    _test_file_time = fs.getFileStatus(test_file_path).getModificationTime();
}

From source file:com.ikanow.aleph2.core.shared.utils.TestJarCacheUtils.java

License:Apache License

@Test
public void test_localFilePresentButOld()
        throws InterruptedException, ExecutionException, AccessControlException, FileAlreadyExistsException,
        FileNotFoundException, ParentNotDirectoryException, IOException {

    final FileContext localfs = FileContext.getLocalFSFileContext(new Configuration());

    String java_name = _globals.local_cached_jar_dir() + File.separator + "testX.cache.jar";
    final String expected_cache_name = java_name.replace(File.separator, "/");
    final Path expected_cache_path = localfs.makeQualified(new Path(expected_cache_name));

    // Just make sure we've deleted the old file
    try {/*from   ww  w  . j a va  2 s .  c o m*/
        System.out.println("Deleted: " + new File(java_name).delete());
    } catch (Exception e) {
        fail("Misc Error: " + e);
    }

    assertTrue("Remote file exists", new File(_test_file_path).exists());
    assertFalse("Local file doesn't exist: " + java_name, new File(java_name).exists());

    // Now create the file

    localfs.create(expected_cache_path, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE));

    localfs.setTimes(expected_cache_path, 0L, 0L);

    // check something has happened:
    assertEquals(0L, localfs.getFileStatus(expected_cache_path).getModificationTime());
    assertNotEquals(0L, _test_file_time);

    // Now run the test routine

    final SharedLibraryBean library_bean = BeanTemplateUtils.build(SharedLibraryBean.class)
            .with(SharedLibraryBean::path_name, _test_file_path).with(SharedLibraryBean::_id, "testX").done()
            .get();

    final Validation<BasicMessageBean, String> ret_val_1 = JarCacheUtils.getCachedJar(
            _globals.local_cached_jar_dir(), library_bean, _mock_hdfs, "testX", new TestMessageBean()).get();

    assertEquals(expected_cache_path.toString(), ret_val_1.success());

    assertTrue("Local file still exists", new File(expected_cache_name).exists());

    assertTrue("File time should have been updated",
            localfs.getFileStatus(expected_cache_path).getModificationTime() >= _test_file_time);
}