Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.aliyun.odps.volume; import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.channels.Channels; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.aliyun.odps.VolumeException; import com.aliyun.odps.fs.VolumeFileSystemConfigKeys; import com.aliyun.odps.volume.protocol.VolumeFSConstants; /** * VolumeFSInputStream * * @author Emerson Zhao [mailto:zhenyi.zzy@alibaba-inc.com] * */ public class VolumeFSInputStream extends FSInputStream { private static final Logger LOG = LoggerFactory.getLogger(VolumeFSInputStream.class); private VolumeFSClient volumeFSClient; private String path; private boolean closed = true; private Long fileLength; private long pos = 0; /** * unit:byte */ private long blockSize; private String uuid; private Map<Long, Integer> cache = new ConcurrentHashMap<Long, Integer>(); private InputStream in; private RandomAccessFile raf; private long curIndex = 0; private File buffer_block_dir; private boolean isSeeked = false; private boolean isCache = false; private boolean seekOptimization; private static final Integer STATUS_DOWNLOADING = 0; private static final Integer STATUS_DOWNLOADED = 1; private ExecutorService executorService; private boolean isPreLoaded = false; public VolumeFSInputStream(String path, VolumeFSClient volumeClient, Long fileLength, Configuration conf) throws IOException { this.path = path; this.volumeFSClient = volumeClient; this.seekOptimization = conf.getBoolean(VolumeFileSystemConfigKeys.ODPS_VOLUME_SEEK_OPTIMIZATION_ENABLED, false); if (this.seekOptimization) { this.blockSize = conf.getLong(VolumeFileSystemConfigKeys.ODPS_VOLUME_BLOCK_SIZE, VolumeFSConstants.DEFAULT_VOLUME_BLOCK_SIZE); } this.fileLength = fileLength; this.closed = false; this.uuid = UUID.randomUUID().toString(); buffer_block_dir = new File(conf.get(VolumeFileSystemConfigKeys.ODPS_VOLUME_BLOCK_BUFFER_DIR, VolumeFSConstants.DEFAULT_VOLUME_BLOCK_BUFFER_DIR)); if (!buffer_block_dir.exists() && !buffer_block_dir.mkdirs()) { throw new IOException("Cannot create Volume block buffer directory: " + buffer_block_dir); } if (seekOptimization) { executorService = Executors.newFixedThreadPool(1); } } @Override public synchronized void seek(long pos) throws IOException { checkClosed(); if (pos < 0) { throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK); } if (pos > fileLength) { throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF); } this.pos = pos; this.isSeeked = true; } @Override public synchronized long getPos() throws IOException { checkClosed(); return pos; } @Override public synchronized boolean seekToNewSource(long targetPos) throws IOException { checkClosed(); seek(targetPos); try { in = getInputStream(); return true; } catch (Exception e) { return false; } } @Override public int read() throws IOException { byte[] buf = new byte[1]; int len = read(buf, 0, 1); if (len == 1) { return (int) buf[0] & 0xFF; } return VolumeFSConstants.EOF; } @Override public int read(byte[] buf, int off, int len) throws IOException { // Inline to optimize performance if (closed) { throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); } if (pos >= fileLength) { return VolumeFSConstants.EOF; } if (in == null || isSeeked || (seekOptimization && isCache && pos / blockSize != curIndex)) { in = getInputStream(); } int avaliable = in.read(buf, off, len); pos = pos + avaliable; if (avaliable == len || pos >= fileLength) { return avaliable; } else { IOUtils.closeQuietly(in); in = null; return avaliable + read(buf, off + avaliable, len - avaliable); } } private synchronized InputStream getInputStream() throws IOException { if (seekOptimization) { long index = pos / blockSize; if (!isPreLoaded) { isPreLoaded = true; seekOptimization(); } if (in != null && index == curIndex && isCache && isSeeked) { in = Channels.newInputStream(raf.getChannel().position(pos - curIndex * blockSize)); return in; } else { IOUtils.closeQuietly(in); IOUtils.closeQuietly(raf); if (cache.get(index) != STATUS_DOWNLOADED) { curIndex = index; return getRemoteInputStream(); } else { File blockFile = new File(buffer_block_dir, getBlockFileName(index)); if (!blockFile.exists()) { cache.remove(index); curIndex = index; return getRemoteInputStream(); } return getLocalInputStream(index); } } } else { IOUtils.closeQuietly(in); return getRemoteInputStream(); } } private synchronized InputStream getLocalInputStream(long index) throws FileNotFoundException, IOException { if (!seekOptimization) { return null; } isCache = true; File blockFile = new File(buffer_block_dir, getBlockFileName(index)); raf = new RandomAccessFile(blockFile, "r"); return Channels.newInputStream(raf.getChannel().position(pos - curIndex * blockSize)); } private synchronized InputStream getRemoteInputStream() throws IOException { isSeeked = false; isCache = false; try { return volumeFSClient.openInputStream(path, pos, fileLength - 1); } catch (VolumeException e) { throw new IOException(e.getMessage(), e); } } private void seekOptimization() { long totalBlockNum = (long) Math.ceil(fileLength.doubleValue() / blockSize); if (totalBlockNum > 2) { for (long i = 2l; i < totalBlockNum; i++) { if (!cache.containsKey(i)) { cache.put(i, STATUS_DOWNLOADING); executorService.submit(new SeekOptimizationWorker(i)); } } } for (long i = totalBlockNum - 1; i >= 0; i--) { if (!cache.containsKey(i)) { cache.put(i, STATUS_DOWNLOADING); executorService.submit(new SeekOptimizationWorker(i)); } } } @Override public synchronized int available() throws IOException { checkClosed(); return (int) (fileLength - pos); } @Override public synchronized void close() { if (!closed) { closed = true; IOUtils.closeQuietly(in); IOUtils.closeQuietly(raf); if (executorService != null) executorService.shutdownNow(); } } private synchronized void checkClosed() throws IOException { if (closed) { throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); } } private String getBlockFileName(long index) { return "volumefs-input-block-" + uuid + "-" + index + ".tmp"; } class SeekOptimizationWorker implements Runnable { private Long index; SeekOptimizationWorker(Long index) { this.index = index; } @Override public void run() { long start = index * blockSize; if (start > fileLength - 1) return; long end = Math.min((index + 1) * blockSize - 1, fileLength - 1); File blockFile = new File(buffer_block_dir, getBlockFileName(index)); blockFile.deleteOnExit(); try { volumeFSClient.downloadFile(path, start, end, blockFile, false); cache.put(index, STATUS_DOWNLOADED); } catch (VolumeException e) { cache.remove(index); LOG.error(e.getMessage(), e); } } } }