Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tajo.yarn; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.cache.RemovalListener; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.Gson; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MutableCounterInt; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryService; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.exception.InvalidURLException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.pullserver.PullServerConstants; import org.apache.tajo.pullserver.PullServerUtil; import org.apache.tajo.pullserver.PullServerUtil.PullServerParams; import org.apache.tajo.pullserver.retriever.FileChunk; import org.apache.tajo.pullserver.retriever.IndexCacheKey; import org.apache.tajo.storage.index.bst.BSTIndex; import org.apache.tajo.storage.index.bst.BSTIndex.BSTIndexReader; import org.apache.tajo.util.SizeOf; import org.apache.tajo.util.TajoIdUtils; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.*; import org.jboss.netty.channel.group.ChannelGroup; import org.jboss.netty.channel.group.DefaultChannelGroup; import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; import org.jboss.netty.handler.codec.http.*; import org.jboss.netty.handler.codec.http.HttpHeaders.Names; import org.jboss.netty.handler.codec.http.HttpHeaders.Values; import org.jboss.netty.handler.ssl.SslHandler; import org.jboss.netty.handler.stream.ChunkedWriteHandler; import org.jboss.netty.util.CharsetUtil; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.*; import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; public class TajoPullServerService extends AuxiliaryService { private static final Log LOG = LogFactory.getLog(TajoPullServerService.class); private TajoConf tajoConf; private int port; private ChannelFactory selector; private final ChannelGroup accepted = new DefaultChannelGroup("Pull server group"); private HttpChannelInitializer channelInitializer; private int sslFileBufferSize; private int maxUrlLength; private ApplicationId appId; private FileSystem localFS; /** * Should the shuffle use posix_fadvise calls to manage the OS cache during * sendfile */ private boolean manageOsCache; private int readaheadLength; private ReadaheadPool readaheadPool = ReadaheadPool.getInstance(); private static final Map<String, String> userRsrc = new ConcurrentHashMap<>(); private String userName; private LoadingCache<IndexCacheKey, BSTIndexReader> indexReaderCache = null; private int lowCacheHitCheckThreshold; @Metrics(name = "PullServerShuffleMetrics", about = "PullServer output metrics", context = "tajo") static class ShuffleMetrics implements ChannelFutureListener { @Metric({ "OutputBytes", "PullServer output in bytes" }) MutableCounterLong shuffleOutputBytes; @Metric({ "Failed", "# of failed shuffle outputs" }) MutableCounterInt shuffleOutputsFailed; @Metric({ "Succeeded", "# of succeeded shuffle outputs" }) MutableCounterInt shuffleOutputsOK; @Metric({ "Connections", "# of current shuffle connections" }) MutableGaugeInt shuffleConnections; @Override public void operationComplete(ChannelFuture future) throws Exception { if (future.isSuccess()) { shuffleOutputsOK.incr(); } else { shuffleOutputsFailed.incr(); } shuffleConnections.decr(); } } final ShuffleMetrics metrics; TajoPullServerService(MetricsSystem ms) { super(PullServerConstants.PULLSERVER_SERVICE_NAME); metrics = ms.register(new ShuffleMetrics()); } @SuppressWarnings("UnusedDeclaration") public TajoPullServerService() { this(DefaultMetricsSystem.instance()); } @Override public void initializeApplication(ApplicationInitializationContext context) { // TODO these bytes should be versioned // TODO: Once SHuffle is out of NM, this can use MR APIs String user = context.getUser(); ApplicationId appId = context.getApplicationId(); // ByteBuffer secret = context.getApplicationDataForService(); userRsrc.put(appId.toString(), user); } @Override public void stopApplication(ApplicationTerminationContext context) { userRsrc.remove(context.getApplicationId().toString()); } // TODO change AbstractService to throw InterruptedException @Override public void serviceInit(Configuration conf) throws Exception { tajoConf = new TajoConf(conf); manageOsCache = tajoConf.getBoolean(PullServerConstants.SHUFFLE_MANAGE_OS_CACHE, PullServerConstants.DEFAULT_SHUFFLE_MANAGE_OS_CACHE); readaheadLength = tajoConf.getInt(PullServerConstants.SHUFFLE_READAHEAD_BYTES, PullServerConstants.DEFAULT_SHUFFLE_READAHEAD_BYTES); int workerNum = tajoConf.getIntVar(ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM); ThreadFactory bossFactory = new ThreadFactoryBuilder().setNameFormat("TajoPullServerService Netty Boss #%d") .build(); ThreadFactory workerFactory = new ThreadFactoryBuilder() .setNameFormat("TajoPullServerService Netty Worker #%d").build(); selector = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory), workerNum); localFS = new LocalFileSystem(); maxUrlLength = tajoConf.getIntVar(ConfVars.PULLSERVER_FETCH_URL_MAX_LENGTH); LOG.info("Tajo PullServer initialized: readaheadLength=" + readaheadLength); ServerBootstrap bootstrap = new ServerBootstrap(selector); try { channelInitializer = new HttpChannelInitializer(tajoConf); } catch (Exception ex) { throw new RuntimeException(ex); } bootstrap.setPipelineFactory(channelInitializer); port = tajoConf.getIntVar(ConfVars.PULLSERVER_PORT); Channel ch = bootstrap.bind(new InetSocketAddress(port)); accepted.add(ch); port = ((InetSocketAddress) ch.getLocalAddress()).getPort(); tajoConf.set(ConfVars.PULLSERVER_PORT.varname, Integer.toString(port)); LOG.info(getName() + " listening on port " + port); sslFileBufferSize = tajoConf.getInt(PullServerConstants.SUFFLE_SSL_FILE_BUFFER_SIZE_KEY, PullServerConstants.DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE); int cacheSize = tajoConf.getIntVar(ConfVars.PULLSERVER_CACHE_SIZE); int cacheTimeout = tajoConf.getIntVar(ConfVars.PULLSERVER_CACHE_TIMEOUT); indexReaderCache = CacheBuilder.newBuilder().maximumSize(cacheSize) .expireAfterWrite(cacheTimeout, TimeUnit.MINUTES).removalListener(removalListener) .build(new CacheLoader<IndexCacheKey, BSTIndexReader>() { @Override public BSTIndexReader load(IndexCacheKey key) throws Exception { return new BSTIndex(tajoConf).getIndexReader(new Path(key.getPath(), "index")); } }); lowCacheHitCheckThreshold = (int) (cacheSize * 0.1f); super.serviceInit(tajoConf); LOG.info("TajoPullServerService started: port=" + port); } @Override public void serviceStop() throws Exception { // TODO: check this wait accepted.close().awaitUninterruptibly(10, TimeUnit.SECONDS); if (selector != null) { ServerBootstrap bootstrap = new ServerBootstrap(selector); bootstrap.releaseExternalResources(); } if (channelInitializer != null) { channelInitializer.destroy(); } localFS.close(); indexReaderCache.invalidateAll(); super.serviceStop(); } @VisibleForTesting public int getPort() { return port; } @Override public ByteBuffer getMetaData() { try { return serializeMetaData(port); } catch (IOException e) { LOG.error("Error during getMeta", e); // TODO add API to AuxiliaryServices to report failures return null; } } /** * Serialize the shuffle port into a ByteBuffer for use later on. * @param port the port to be sent to the ApplciationMaster * @return the serialized form of the port. */ public static ByteBuffer serializeMetaData(int port) throws IOException { //TODO these bytes should be versioned return ByteBuffer.allocate(SizeOf.SIZE_OF_INT).putInt(port); } class HttpChannelInitializer implements ChannelPipelineFactory { final PullServer PullServer; private SSLFactory sslFactory; public HttpChannelInitializer(TajoConf conf) throws Exception { PullServer = new PullServer(conf); if (conf.getBoolVar(ConfVars.SHUFFLE_SSL_ENABLED_KEY)) { sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); sslFactory.init(); } } public void destroy() { if (sslFactory != null) { sslFactory.destroy(); } } @Override public ChannelPipeline getPipeline() throws Exception { ChannelPipeline pipeline = Channels.pipeline(); if (sslFactory != null) { pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine())); } int maxChunkSize = getConfig().getInt(ConfVars.SHUFFLE_FETCHER_CHUNK_MAX_SIZE.varname, ConfVars.SHUFFLE_FETCHER_CHUNK_MAX_SIZE.defaultIntVal); pipeline.addLast("codec", new HttpServerCodec(maxUrlLength, 8192, maxChunkSize)); pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16)); pipeline.addLast("chunking", new ChunkedWriteHandler()); pipeline.addLast("shuffle", PullServer); return pipeline; // TODO factor security manager into pipeline // TODO factor out encode/decode to permit binary shuffle // TODO factor out decode of index to permit alt. models } } @ChannelHandler.Sharable class PullServer extends SimpleChannelUpstreamHandler { private final TajoConf conf; private final LocalDirAllocator lDirAlloc = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); private final Gson gson = new Gson(); public PullServer(TajoConf conf) throws IOException { this.conf = conf; // init local temporal dir lDirAlloc.getAllLocalPathsToRead(".", conf); } @Override public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent evt) throws Exception { accepted.add(evt.getChannel()); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Current number of shuffle connections (%d)", accepted.size())); } super.channelOpen(ctx, evt); } @Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt) throws Exception { HttpRequest request = (HttpRequest) evt.getMessage(); Channel ch = evt.getChannel(); if (request.getMethod() == HttpMethod.DELETE) { HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NO_CONTENT); ch.write(response).addListener(ChannelFutureListener.CLOSE); clearIndexCache(request.getUri()); return; } else if (request.getMethod() != HttpMethod.GET) { sendError(ctx, HttpResponseStatus.METHOD_NOT_ALLOWED); return; } // Parsing the URL into key-values try { final PullServerParams params = new PullServerParams(request.getUri()); if (PullServerUtil.isChunkRequest(params.requestType())) { handleChunkRequest(ctx, request, params); } else { handleMetaRequest(ctx, request, params); } } catch (Throwable e) { LOG.error("Failed to handle request " + request.getUri()); sendError(ctx, e.getMessage(), HttpResponseStatus.BAD_REQUEST); return; } } /** * Upon a request from TajoWorker, this method clears index cache for fetching data of an execution block. * It is called whenever an execution block is completed. * * @param uri query URI which indicates the execution block id * @throws IOException * @throws InvalidURLException */ public void clearIndexCache(String uri) throws IOException, InvalidURLException { // Simply parse the given uri String[] tokens = uri.split("="); if (tokens.length != 2 || !tokens[0].equals("ebid")) { throw new IllegalArgumentException("invalid params: " + uri); } ExecutionBlockId ebId = TajoIdUtils.createExecutionBlockId(tokens[1]); String queryId = ebId.getQueryId().toString(); String ebSeqId = Integer.toString(ebId.getId()); List<IndexCacheKey> removed = new ArrayList<>(); synchronized (indexReaderCache) { for (Entry<IndexCacheKey, BSTIndexReader> e : indexReaderCache.asMap().entrySet()) { IndexCacheKey key = e.getKey(); if (key.getQueryId().equals(queryId) && key.getEbSeqId().equals(ebSeqId)) { e.getValue().forceClose(); removed.add(e.getKey()); } } indexReaderCache.invalidateAll(removed); } removed.clear(); synchronized (waitForRemove) { for (Entry<IndexCacheKey, BSTIndexReader> e : waitForRemove.entrySet()) { IndexCacheKey key = e.getKey(); if (key.getQueryId().equals(queryId) && key.getEbSeqId().equals(ebSeqId)) { e.getValue().forceClose(); removed.add(e.getKey()); } } for (IndexCacheKey eachKey : removed) { waitForRemove.remove(eachKey); } } } private void handleMetaRequest(ChannelHandlerContext ctx, HttpRequest request, final PullServerParams params) throws IOException, ExecutionException { final List<String> jsonMetas; try { jsonMetas = PullServerUtil.getJsonMeta(conf, lDirAlloc, localFS, params, gson, indexReaderCache, lowCacheHitCheckThreshold); } catch (FileNotFoundException e) { sendError(ctx, e.getMessage(), HttpResponseStatus.NO_CONTENT); return; } catch (IOException | IllegalArgumentException e) { // IOException, EOFException, IllegalArgumentException sendError(ctx, e.getMessage(), HttpResponseStatus.BAD_REQUEST); return; } catch (ExecutionException e) { // There are some problems in index cache throw new TajoInternalError(e.getCause()); } HttpResponse response = new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.OK); response.setContent(ChannelBuffers.copiedBuffer(gson.toJson(jsonMetas), CharsetUtil.UTF_8)); response.setHeader(Names.CONTENT_TYPE, "application/json; charset=UTF-8"); HttpHeaders.setContentLength(response, response.getContent().readableBytes()); if (HttpHeaders.isKeepAlive(request)) { response.setHeader(Names.CONNECTION, Values.KEEP_ALIVE); } ChannelFuture writeFuture = ctx.getChannel().write(response); // Decide whether to close the connection or not. if (!HttpHeaders.isKeepAlive(request)) { // Close the connection when the whole content is written out. writeFuture.addListener(ChannelFutureListener.CLOSE); } } private void handleChunkRequest(ChannelHandlerContext ctx, HttpRequest request, final PullServerParams params) throws IOException { final List<FileChunk> chunks; try { chunks = PullServerUtil.getFileChunks(conf, lDirAlloc, localFS, params, indexReaderCache, lowCacheHitCheckThreshold); } catch (FileNotFoundException e) { sendError(ctx, e.getMessage(), HttpResponseStatus.NO_CONTENT); return; } catch (IOException | IllegalArgumentException e) { // IOException, EOFException, IllegalArgumentException sendError(ctx, e.getMessage(), HttpResponseStatus.BAD_REQUEST); return; } catch (ExecutionException e) { // There are some problems in index cache throw new TajoInternalError(e.getCause()); } // Write the content. final Channel ch = ctx.getChannel(); if (chunks.size() == 0) { HttpResponse response = new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.NO_CONTENT); if (!HttpHeaders.isKeepAlive(request)) { ch.write(response).addListener(ChannelFutureListener.CLOSE); } else { response.setHeader(Names.CONNECTION, Values.KEEP_ALIVE); ch.write(response); } } else { FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]); ChannelFuture writeFuture = null; HttpResponse response = new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.OK); long totalSize = 0; StringBuilder sb = new StringBuilder(); for (FileChunk chunk : file) { totalSize += chunk.length(); sb.append(Long.toString(chunk.length())).append(","); } sb.deleteCharAt(sb.length() - 1); HttpHeaders.addHeader(response, PullServerConstants.CHUNK_LENGTH_HEADER_NAME, sb.toString()); HttpHeaders.setContentLength(response, totalSize); if (HttpHeaders.isKeepAlive(request)) { response.setHeader(Names.CONNECTION, Values.KEEP_ALIVE); } // Write the initial line and the header. writeFuture = ch.write(response); for (FileChunk chunk : file) { writeFuture = sendFile(ctx, chunk); if (writeFuture == null) { sendError(ctx, HttpResponseStatus.NOT_FOUND); return; } } // Decide whether to close the connection or not. if (!HttpHeaders.isKeepAlive(request)) { // Close the connection when the whole content is written out. writeFuture.addListener(ChannelFutureListener.CLOSE); } } } private ChannelFuture sendFile(ChannelHandlerContext ctx, FileChunk file) throws IOException { Channel ch = ctx.getChannel(); RandomAccessFile spill = null; ChannelFuture writeFuture; try { spill = new RandomAccessFile(file.getFile(), "r"); if (ctx.getPipeline().get(SslHandler.class) == null) { final FadvisedFileRegion filePart = new FadvisedFileRegion(spill, file.startOffset(), file.length(), manageOsCache, readaheadLength, readaheadPool, file.getFile().getAbsolutePath()); writeFuture = ch.write(filePart); writeFuture.addListener(new FileCloseListener(filePart)); } else { // HTTPS cannot be done with zero copy. final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, file.startOffset(), file.length(), sslFileBufferSize, manageOsCache, readaheadLength, readaheadPool, file.getFile().getAbsolutePath()); writeFuture = ch.write(chunk); } } catch (FileNotFoundException e) { LOG.fatal(file.getFile() + " not found"); return null; } catch (Throwable e) { LOG.fatal("error while sending a file: ", e); if (spill != null) { //should close a opening file LOG.warn("Close the file " + file.getFile().getAbsolutePath()); spill.close(); } return null; } metrics.shuffleConnections.incr(); metrics.shuffleOutputBytes.incr(file.length()); // optimistic return writeFuture; } private void sendError(ChannelHandlerContext ctx, HttpResponseStatus status) { sendError(ctx, "", status); } private void sendError(ChannelHandlerContext ctx, String message, HttpResponseStatus status) { HttpResponse response = new DefaultHttpResponse(HTTP_1_1, status); response.setHeader(Names.CONTENT_TYPE, "text/plain; charset=UTF-8"); // Put shuffle version into http header ChannelBuffer content = ChannelBuffers.copiedBuffer(message, CharsetUtil.UTF_8); response.setContent(content); response.setHeader(Names.CONTENT_LENGTH, content.writerIndex()); // Close the connection as soon as the error message is sent. ctx.getChannel().write(response).addListener(ChannelFutureListener.CLOSE); } @Override public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { Channel ch = e.getChannel(); Throwable cause = e.getCause(); LOG.error(cause.getMessage(), cause); if (ch.isOpen()) { ch.close(); } } } // Temporal space to wait for the completion of all index lookup operations private final ConcurrentHashMap<IndexCacheKey, BSTIndexReader> waitForRemove = new ConcurrentHashMap<>(); // RemovalListener is triggered when an item is removed from the index reader cache. // It closes index readers when they are not used anymore. // If they are still being used, they are moved to waitForRemove map to wait for other operations' completion. private final RemovalListener<IndexCacheKey, BSTIndexReader> removalListener = (removal) -> { BSTIndexReader reader = removal.getValue(); if (reader.getReferenceNum() == 0) { try { reader.close(); // tear down properly } catch (IOException e) { throw new RuntimeException(e); } waitForRemove.remove(removal.getKey()); } else { waitForRemove.put(removal.getKey(), reader); } }; }