Example usage for com.google.common.io ByteSource ByteSource

List of usage examples for com.google.common.io ByteSource ByteSource

  1. HOME
  2. Java
  3. com.google.common
  4. com.google.common.io.*
  5. ByteSource
  6. ByteSource

Introduction

In this page you can find the example usage for com.google.common.io ByteSource ByteSource.

Prototype

protected ByteSource()

Source Link

Document

Constructor for use by subclasses.

Usage

From source file:io.druid.storage.oss.OssDataSegmentPuller.java

public FileUtils.FileCopyResult getSegmentFiles(final URI uri, final File outDir)
        throws SegmentLoadingException {

    //  get bucket and key from uri
    final String bucket = getBucket(uri);
    final String key = getKey(uri);

    //  get index from oss
    log.info("pulling index from oss key [%s] to out dir [%s]", key, outDir);
    if (!ossClient.doesObjectExist(bucket, key)) {
        throw new SegmentLoadingException("index file [%s] does not exist", key);
    }// w  w  w.j ava  2 s.  c o m

    //  create out directory
    outDir.mkdirs();
    if (!outDir.isDirectory()) {
        throw new ISE("out dir [%s] must be a directory", outDir);
    }

    try {

        //  get index input stream
        final ByteSource byteSource = new ByteSource() {
            @Override
            public InputStream openStream() throws IOException {
                return ossClient.getObject(bucket, key).getObjectContent();
            }
        };

        //  if zip file
        if (CompressionUtils.isZip(key)) {
            final FileUtils.FileCopyResult result = CompressionUtils.unzip(byteSource, outDir,
                    OssUtils.OssRETRY, true);
            log.info("loaded %d bytes from oss [%s] to [%s]", result.size(), key, outDir.getAbsolutePath());
            return result;
        }

        //   if gz file
        if (CompressionUtils.isGz(key)) {
            final File outFile = new File(outDir, Files.getNameWithoutExtension(uri.getPath()));
            final FileUtils.FileCopyResult result = CompressionUtils.gunzip(byteSource, outFile,
                    OssUtils.OssRETRY);
            log.info("loaded %d bytes from oss [%s] to [%s]", result.size(), key, outFile.getAbsolutePath());
            return result;
        }

        throw new IAE("do not know how to load file type at [%s]", uri.toString());

    } catch (Exception e) {

        try {

            //  delete out directory
            org.apache.commons.io.FileUtils.deleteDirectory(outDir);

        } catch (IOException ioe) {
            log.warn(ioe, "failed to remove output directory [%s] for segment pulled from [%s]",
                    outDir.getAbsolutePath(), uri.getPath());
        }

        throw new SegmentLoadingException(e, e.getMessage());
    }
}

From source file:com.facebook.buck.httpserver.ArtifactCacheHandler.java

private int handleGet(Request baseRequest, HttpServletResponse response) throws IOException {
    if (!artifactCache.isPresent()) {
        response.getWriter().write("Serving local cache is disabled for this instance.");
        return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
    }//  www. j  a  va2  s.c o  m

    String path = baseRequest.getUri().getPath();
    String[] pathElements = path.split("/");
    if (pathElements.length != 4 || !pathElements[2].equals("key")) {
        response.getWriter().write("Incorrect url format.");
        return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
    }

    RuleKey ruleKey = new RuleKey(pathElements[3]);

    Path temp = null;
    try {
        projectFilesystem.mkdirs(projectFilesystem.getBuckPaths().getScratchDir());
        temp = projectFilesystem.createTempFile(projectFilesystem.getBuckPaths().getScratchDir(),
                "outgoing_rulekey", ".tmp");
        CacheResult fetchResult = artifactCache.get().fetch(ruleKey, LazyPath.ofInstance(temp));
        if (!fetchResult.getType().isSuccess()) {
            return HttpServletResponse.SC_NOT_FOUND;
        }

        final Path tempFinal = temp;
        HttpArtifactCacheBinaryProtocol.FetchResponse fetchResponse = new HttpArtifactCacheBinaryProtocol.FetchResponse(
                ImmutableSet.of(ruleKey), fetchResult.getMetadata(), new ByteSource() {
                    @Override
                    public InputStream openStream() throws IOException {
                        return projectFilesystem.newFileInputStream(tempFinal);
                    }
                });
        fetchResponse.write(response.getOutputStream());
        response.setContentLengthLong(fetchResponse.getContentLength());
        return HttpServletResponse.SC_OK;
    } finally {
        if (temp != null) {
            projectFilesystem.deleteFileAtPathIfExists(temp);
        }
    }
}

From source file:com.facebook.buck.jvm.java.AccumulateClassNamesStep.java

/**
 * @return an Optional that will be absent if there was an error.
 *//* www.  java  2s.  co m*/
public static Optional<ImmutableSortedMap<String, HashCode>> calculateClassHashes(ExecutionContext context,
        ProjectFilesystem filesystem, Path path) {
    final Map<String, HashCode> classNames = new HashMap<>();

    ClasspathTraversal traversal = new ClasspathTraversal(Collections.singleton(path), filesystem) {
        @Override
        public void visit(final FileLike fileLike) throws IOException {
            // When traversing a JAR file, it may have resources or directory entries that do not
            // end in .class, which should be ignored.
            if (!FileLikes.isClassFile(fileLike)) {
                return;
            }

            String key = FileLikes.getFileNameWithoutClassSuffix(fileLike);
            ByteSource input = new ByteSource() {
                @Override
                public InputStream openStream() throws IOException {
                    return fileLike.getInput();
                }
            };
            HashCode value = input.hash(Hashing.sha1());
            HashCode existing = classNames.putIfAbsent(key, value);
            if (existing != null && !existing.equals(value)) {
                throw new IllegalArgumentException(String.format(
                        "Multiple entries with same key but differing values: %1$s=%2$s and %1$s=%3$s", key,
                        value, existing));
            }
        }
    };

    try {
        new DefaultClasspathTraverser().traverse(traversal);
    } catch (IOException e) {
        context.logError(e, "Error accumulating class names for %s.", path);
        return Optional.empty();
    }

    return Optional.of(ImmutableSortedMap.copyOf(classNames, Ordering.natural()));
}

From source file:org.apache.druid.server.lookup.namespace.UriCacheGenerator.java

@Override
@Nullable//from   w w w. j a  va2  s .c om
public CacheScheduler.VersionedCache generateCache(final UriExtractionNamespace extractionNamespace,
        final CacheScheduler.EntryImpl<UriExtractionNamespace> entryId, @Nullable final String lastVersion,
        final CacheScheduler scheduler) throws Exception {
    final boolean doSearch = extractionNamespace.getUriPrefix() != null;
    final URI originalUri = doSearch ? extractionNamespace.getUriPrefix() : extractionNamespace.getUri();
    final SearchableVersionedDataFinder<URI> pullerRaw = pullers.get(originalUri.getScheme());
    if (pullerRaw == null) {
        throw new IAE("Unknown loader type[%s].  Known types are %s", originalUri.getScheme(),
                pullers.keySet());
    }
    if (!(pullerRaw instanceof URIDataPuller)) {
        throw new IAE("Cannot load data from location [%s]. Data pulling from [%s] not supported", originalUri,
                originalUri.getScheme());
    }
    final URIDataPuller puller = (URIDataPuller) pullerRaw;
    final URI uri;
    if (doSearch) {
        final Pattern versionRegex;

        if (extractionNamespace.getFileRegex() != null) {
            versionRegex = Pattern.compile(extractionNamespace.getFileRegex());
        } else {
            versionRegex = null;
        }
        uri = pullerRaw.getLatestVersion(extractionNamespace.getUriPrefix(), versionRegex);

        if (uri == null) {
            throw new FileNotFoundException(
                    StringUtils.format("Could not find match for pattern `%s` in [%s] for %s", versionRegex,
                            originalUri, extractionNamespace));
        }
    } else {
        uri = extractionNamespace.getUri();
    }

    final String uriPath = uri.getPath();

    return RetryUtils.retry(() -> {
        final String version = puller.getVersion(uri);
        try {
            // Important to call equals() against version because lastVersion could be null
            if (version.equals(lastVersion)) {
                log.debug("URI [%s] for [%s] has the same last modified time [%s] as the last cached. "
                        + "Skipping ", uri.toString(), entryId, version);
                return null;
            }
        } catch (NumberFormatException ex) {
            log.debug(ex, "Failed to get last modified timestamp. Assuming no timestamp");
        }
        final ByteSource source = new ByteSource() {
            @Override
            public InputStream openStream() throws IOException {
                return CompressionUtils.decompress(puller.getInputStream(uri), uri.getPath());
            }
        };

        final CacheScheduler.VersionedCache versionedCache = scheduler.createVersionedCache(entryId, version);
        try {
            final long startNs = System.nanoTime();
            final MapPopulator.PopulateResult populateResult = new MapPopulator<>(
                    extractionNamespace.getNamespaceParseSpec().getParser()).populate(source,
                            versionedCache.getCache());
            final long duration = System.nanoTime() - startNs;
            log.info("Finished loading %,d values from %,d lines for [%s] in %,d ns",
                    populateResult.getEntries(), populateResult.getLines(), entryId, duration);
            return versionedCache;
        } catch (Throwable t) {
            try {
                versionedCache.close();
            } catch (Exception e) {
                t.addSuppressed(e);
            }
            throw t;
        }
    }, puller.shouldRetryPredicate(), DEFAULT_NUM_RETRIES);
}

From source file:org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.java

public static ByteSource fileToByteSource(final File file) {
    return new ByteSource() {
        @Override// w  w  w  . j  a  va  2 s . c o  m
        public InputStream openStream() throws IOException {
            return new NamedFileInputStream(file, file.getAbsolutePath());
        }
    };
}

From source file:io.druid.server.lookup.namespace.UriCacheGenerator.java

@Override
@Nullable//ww  w.  ja v  a  2  s  .c  o m
public CacheScheduler.VersionedCache generateCache(final UriExtractionNamespace extractionNamespace,
        final CacheScheduler.EntryImpl<UriExtractionNamespace> entryId, @Nullable final String lastVersion,
        final CacheScheduler scheduler) throws Exception {
    final boolean doSearch = extractionNamespace.getUriPrefix() != null;
    final URI originalUri = doSearch ? extractionNamespace.getUriPrefix() : extractionNamespace.getUri();
    final SearchableVersionedDataFinder<URI> pullerRaw = pullers.get(originalUri.getScheme());
    if (pullerRaw == null) {
        throw new IAE("Unknown loader type[%s].  Known types are %s", originalUri.getScheme(),
                pullers.keySet());
    }
    if (!(pullerRaw instanceof URIDataPuller)) {
        throw new IAE("Cannot load data from location [%s]. Data pulling from [%s] not supported", originalUri,
                originalUri.getScheme());
    }
    final URIDataPuller puller = (URIDataPuller) pullerRaw;
    final URI uri;
    if (doSearch) {
        final Pattern versionRegex;

        if (extractionNamespace.getFileRegex() != null) {
            versionRegex = Pattern.compile(extractionNamespace.getFileRegex());
        } else {
            versionRegex = null;
        }
        uri = pullerRaw.getLatestVersion(extractionNamespace.getUriPrefix(), versionRegex);

        if (uri == null) {
            throw new FileNotFoundException(
                    StringUtils.format("Could not find match for pattern `%s` in [%s] for %s", versionRegex,
                            originalUri, extractionNamespace));
        }
    } else {
        uri = extractionNamespace.getUri();
    }

    final String uriPath = uri.getPath();

    return RetryUtils.retry(() -> {
        final String version = puller.getVersion(uri);
        try {
            // Important to call equals() against version because lastVersion could be null
            if (version.equals(lastVersion)) {
                log.debug("URI [%s] for [%s] has the same last modified time [%s] as the last cached. "
                        + "Skipping ", uri.toString(), entryId, version);
                return null;
            }
        } catch (NumberFormatException ex) {
            log.debug(ex, "Failed to get last modified timestamp. Assuming no timestamp");
        }
        final ByteSource source;
        if (CompressionUtils.isGz(uriPath)) {
            // Simple gzip stream
            log.debug("Loading gz");
            source = new ByteSource() {
                @Override
                public InputStream openStream() throws IOException {
                    return CompressionUtils.gzipInputStream(puller.getInputStream(uri));
                }
            };
        } else {
            source = new ByteSource() {
                @Override
                public InputStream openStream() throws IOException {
                    return puller.getInputStream(uri);
                }
            };
        }

        final CacheScheduler.VersionedCache versionedCache = scheduler.createVersionedCache(entryId, version);
        try {
            final long startNs = System.nanoTime();
            final MapPopulator.PopulateResult populateResult = new MapPopulator<>(
                    extractionNamespace.getNamespaceParseSpec().getParser()).populate(source,
                            versionedCache.getCache());
            final long duration = System.nanoTime() - startNs;
            log.info("Finished loading %,d values from %,d lines for [%s] in %,d ns",
                    populateResult.getEntries(), populateResult.getLines(), entryId, duration);
            return versionedCache;
        } catch (Throwable t) {
            try {
                versionedCache.close();
            } catch (Exception e) {
                t.addSuppressed(e);
            }
            throw t;
        }
    }, puller.shouldRetryPredicate(), DEFAULT_NUM_RETRIES);
}

From source file:io.druid.server.lookup.namespace.URIExtractionNamespaceCacheFactory.java

@Override
public String populateCache(final String id, final URIExtractionNamespace extractionNamespace,
        @Nullable final String lastVersion, final Map<String, String> cache) throws Exception {
    final boolean doSearch = extractionNamespace.getUriPrefix() != null;
    final URI originalUri = doSearch ? extractionNamespace.getUriPrefix() : extractionNamespace.getUri();
    final SearchableVersionedDataFinder<URI> pullerRaw = pullers.get(originalUri.getScheme());
    if (pullerRaw == null) {
        throw new IAE("Unknown loader type[%s].  Known types are %s", originalUri.getScheme(),
                pullers.keySet());/*  w  w w. j  a  v  a 2  s  . c  o  m*/
    }
    if (!(pullerRaw instanceof URIDataPuller)) {
        throw new IAE("Cannot load data from location [%s]. Data pulling from [%s] not supported", originalUri,
                originalUri.getScheme());
    }
    final URIDataPuller puller = (URIDataPuller) pullerRaw;
    final URI uri;
    if (doSearch) {
        final Pattern versionRegex;

        if (extractionNamespace.getFileRegex() != null) {
            versionRegex = Pattern.compile(extractionNamespace.getFileRegex());
        } else {
            versionRegex = null;
        }
        uri = pullerRaw.getLatestVersion(extractionNamespace.getUriPrefix(), versionRegex);

        if (uri == null) {
            throw new FileNotFoundException(
                    String.format("Could not find match for pattern `%s` in [%s] for %s", versionRegex,
                            originalUri, extractionNamespace));
        }
    } else {
        uri = extractionNamespace.getUri();
    }

    final String uriPath = uri.getPath();

    return RetryUtils.retry(new Callable<String>() {
        @Override
        public String call() throws Exception {
            final String version = puller.getVersion(uri);
            try {
                // Important to call equals() against version because lastVersion could be null
                if (version.equals(lastVersion)) {
                    log.debug(
                            "URI [%s] for namespace [%s] has the same last modified time [%s] as the last cached. "
                                    + "Skipping ",
                            uri.toString(), id, version);
                    return lastVersion;
                }
            } catch (NumberFormatException ex) {
                log.debug(ex, "Failed to get last modified timestamp. Assuming no timestamp");
            }
            final ByteSource source;
            if (CompressionUtils.isGz(uriPath)) {
                // Simple gzip stream
                log.debug("Loading gz");
                source = new ByteSource() {
                    @Override
                    public InputStream openStream() throws IOException {
                        return CompressionUtils.gzipInputStream(puller.getInputStream(uri));
                    }
                };
            } else {
                source = new ByteSource() {
                    @Override
                    public InputStream openStream() throws IOException {
                        return puller.getInputStream(uri);
                    }
                };
            }
            final MapPopulator.PopulateResult populateResult = new MapPopulator<>(
                    extractionNamespace.getNamespaceParseSpec().getParser()).populate(source, cache);
            log.info("Finished loading %,d values from %,d lines for namespace [%s]",
                    populateResult.getEntries(), populateResult.getLines(), id);
            return version;
        }
    }, puller.shouldRetryPredicate(), DEFAULT_NUM_RETRIES);
}

From source file:org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.java

public static Collection<ByteSource> filesToByteSources(final Collection<File> streams)
        throws FileNotFoundException {
    return Collections2.transform(streams, new Function<File, ByteSource>() {
        @Override//from w w  w  .  ja  va 2  s .  c  o m
        public ByteSource apply(final File input) {
            return new ByteSource() {
                @Override
                public InputStream openStream() throws IOException {
                    return new NamedFileInputStream(input, input.getAbsolutePath());
                }
            };
        }
    });
}

From source file:org.vaadin.rise.codegen.helpers.JavaFileObjects.java

static ByteSource asByteSource(final JavaFileObject javaFileObject) {
    return new ByteSource() {
        @Override/*from   ww  w  .ja v  a  2s . c  o  m*/
        public InputStream openStream() throws IOException {
            return javaFileObject.openInputStream();
        }
    };
}

From source file:com.eclipsesource.connect.mail.MessageBuilder.java

private byte[] createAttachementContent(Attachement attachement) throws IOException {
    return new ByteSource() {

        @Override//from  w  ww. j  a v  a 2  s  .  c om
        public InputStream openStream() throws IOException {
            return attachement.getContent();
        }
    }.read();
}