List of usage examples for java.util.concurrent LinkedBlockingDeque put
public void put(E e) throws InterruptedException
From source file:com.edgenius.wiki.webapp.action.SpaceAction.java
/** * Redeploy shell - request spaceUname as input parameter * @return// w ww . j a va 2s . c om */ public String shellDeploy() { securityDummy.checkSpaceAdmin(spaceUname); Space space = spaceService.getSpaceByUname(spaceUname); if (space.isPrivate() || space.containExtLinkType(Space.EXT_LINK_SHELL_DISABLED) || StringUtils.equalsIgnoreCase(SharedConstants.SYSTEM_SPACEUNAME, space.getUnixName())) { getRequest().setAttribute("error", "Deploy failed: Space is private or shell service is disabled."); return RET_SHELL; } User anonymous = userReadingService.getUserByName(null); final LinkedBlockingDeque<String[]> pageQ = new LinkedBlockingDeque<String[]>(); //This will make GW request Shell.key again - so if Shell site cleans the data, re-deploy still can work out. //If shell data is still there, it will return same key as GW instanceID and address won't changed. Shell.key = null; new Thread(new Runnable() { public void run() { int pageCount = 0; //in shell side, page request also verify if space exists or not, if not, it will do space request //however, if space already exist, it won't refresh space - if space has menu, it won't refresh again. //so, we also refresh space Shell.notifySpaceCreate(spaceUname); do { try { String[] str = pageQ.take(); if (StringUtils.equalsIgnoreCase(SharedConstants.SYSTEM_SPACEUNAME, str[0])) break; Shell.notifyPageCreate(str[0], str[1], true); pageCount++; //don't explode the too many concurrent request to Shell! Thread.sleep(1000); if ((pageCount % QUOTA_RESET_COUNT) == 0) { log.warn( "Maximumn page shell request count arrived {}, sleep another 24 hours on space {}.", QUOTA_RESET_COUNT, spaceUname); //google app engine has quota limitation. Here will sleep 24 hours to wait next quota reset. Thread.sleep(24 * 3600 * 1000); log.warn( "Maximumn page shell request sleep is end, restart page request process on space{}.", spaceUname); } } catch (InterruptedException e) { log.error("Thread interrupted for shell request", e); } } while (true); log.info("Shell page request is done for {} pages on space {}.", pageCount, spaceUname); } }).start(); int pageCount = 0; long start = System.currentTimeMillis(); log.info("Shell on space {} redeploy request starting...", spaceUname); List<String> pages = pageService.getPagesUuidInSpace(spaceUname, anonymous); if (pages != null) { for (String puuid : pages) { try { pageQ.put(new String[] { spaceUname, puuid }); pageCount++; } catch (InterruptedException e) { log.error("Thread interrupted for shell Page Queue on space " + spaceUname, e); } } } log.info("All shell on space {} request put into queue. Pages{}; Takes {}s", new Object[] { spaceUname, pageCount, (System.currentTimeMillis() - start) / 1000 }); try { pageQ.put(new String[] { SharedConstants.SYSTEM_SPACEUNAME, "" }); } catch (InterruptedException e) { log.error("Thread interrupted for shell Page Queue - end sign", e); } getRequest().setAttribute("message", "Shell is redeploying, this may take a couple of minutes to complete."); return shell(); }
From source file:com.edgenius.wiki.webapp.admin.action.AdvanceAdminAction.java
public String redeployShell() { User anonymous = userReadingService.getUserByName(null); //in shell side, page request also verify if space exists or not, if not, it will do space request //so here won't do space request final LinkedBlockingDeque<String[]> pageQ = new LinkedBlockingDeque<String[]>(); //This will make GW request Shell.key again - so if Shell site cleans the data, re-deploy still can work out. //If shell data is still there, it will return same key as GW instanceID and address won't changed. Shell.key = null;/*from w ww .j a v a2 s. c o m*/ new Thread(new Runnable() { public void run() { int pageCount = 0; do { try { String[] str = pageQ.take(); if (StringUtils.equalsIgnoreCase(SharedConstants.SYSTEM_SPACEUNAME, str[0])) break; Shell.notifyPageCreate(str[0], str[1], true); pageCount++; //don't explode the too many concurrent request to Shell! Thread.sleep(1000); if ((pageCount % QUOTA_RESET_COUNT) == 0) { log.warn("Maximumn page shell request count arrived {}, sleep another 24 hours ", QUOTA_RESET_COUNT); //google app engine has quota limitation. Here will sleep 24 hours to wait next quota reset. Thread.sleep(24 * 3600 * 1000); log.warn("Maximumn page shell request sleep is end, restart page request process"); } } catch (InterruptedException e) { log.error("Thread interrupted for shell request", e); } } while (true); ActivityLog activity = new ActivityLog(); activity.setType(ActivityType.Type.SYSTEM_EVENT.getCode()); activity.setSubType(ActivityType.SubType.REDEPLOY_SHELL.getCode()); activity.setTgtResourceName("SHELL-DEPLOYED");//hardcode activity.setCreatedDate(new Date()); activityLog.save(activity); log.info("Shell page request is done for {} pages", pageCount); } }).start(); int pageCount = 0; long start = System.currentTimeMillis(); log.info("Shell redeploy request starting..."); List<Space> spaces = spaceDAO.getObjects(); if (spaces != null) { for (Space space : spaces) { if (space.isPrivate() || space.containExtLinkType(Space.EXT_LINK_SHELL_DISABLED) || StringUtils.equalsIgnoreCase(SharedConstants.SYSTEM_SPACEUNAME, space.getUnixName())) continue; String spaceUname = space.getUnixName(); List<String> pages = pageDAO.getPagesUuidInSpace(spaceUname); if (pages != null) { for (String puuid : pages) { if (!securityService.isAllowPageReading(spaceUname, puuid, anonymous)) continue; try { pageQ.put(new String[] { spaceUname, puuid }); pageCount++; } catch (InterruptedException e) { log.error("Thread interrupted for shell Page Queue", e); } } } } log.info("All shell request put into queue. Pages{}; Takes {}s", new Object[] { pageCount, (System.currentTimeMillis() - start) / 1000 }); } try { pageQ.put(new String[] { SharedConstants.SYSTEM_SPACEUNAME, "" }); } catch (InterruptedException e) { log.error("Thread interrupted for shell Page Queue - end sign", e); } getRequest().setAttribute("message", messageService.getMessage("redeploy.shell.invoked")); return MESSAGE; }
From source file:org.commoncrawl.service.listcrawler.DataTransferAgent.java
static int uploadSingeFile(CCBridgeServerMapping mapping, FileSystem fs, Configuration conf, Path hdfsFilePath, String uploadName, EventLoop eventLoop) throws IOException { final FileStatus fileStatus = fs.getFileStatus(hdfsFilePath); LOG.info("Uploading:" + uploadName + " size:" + fileStatus.getLen() + " to:" + mapping._internalName); {/*from w w w .j a v a 2 s . c om*/ // construct url URL fePostURL = new URL("http://" + mapping._externalName + ":8090/"); LOG.info("POST URL IS:" + fePostURL.toString()); // open input stream final FSDataInputStream is = fs.open(hdfsFilePath); final Semaphore blockingSemaphore = new Semaphore(0); NIOHttpConnection connection = null; try { // create connection connection = new NIOHttpConnection(fePostURL, eventLoop.getSelector(), eventLoop.getResolver(), null); // set listener connection.setListener(new Listener() { @Override public void HttpConnectionStateChanged(NIOHttpConnection theConnection, State oldState, State state) { LOG.info("Connection State Changed to:" + state.toString()); if (state == State.DONE || state == State.ERROR) { //LOG.info("Connection Transition to Done or Error"); //LOG.info("Response Headers:" + theConnection.getResponseHeaders().toString()); blockingSemaphore.release(); } } @Override public void HttpContentAvailable(NIOHttpConnection theConnection, NIOBufferList contentBuffer) { // TODO Auto-generated method stub } }); // set headers connection.getRequestHeaders().reset(); connection.getRequestHeaders().prepend("PUT /put?src=" + uploadName + " HTTP/1.1", null); connection.getRequestHeaders().set("Host", mapping._internalName + ":8090"); connection.getRequestHeaders().set("Content-Length", Long.toString(fileStatus.getLen())); connection.getRequestHeaders().set("Connection", "keep-alive"); connection.setPopulateDefaultHeaderItems(false); final LinkedBlockingDeque<BufferStruct> _loaderQueue = new LinkedBlockingDeque<BufferStruct>(20); final AtomicBoolean eof = new AtomicBoolean(); final ByteBuffer sentinel = ByteBuffer.allocate(4096); sentinel.position(sentinel.position()); final Thread loaderThread = new Thread(new Runnable() { int _id = 0; @Override public void run() { int bytesRead; byte incomingBuffer[] = new byte[4096 * 10]; try { while ((bytesRead = is.read(incomingBuffer)) != -1) { ByteBuffer buffer = ByteBuffer.wrap(incomingBuffer, 0, bytesRead); buffer.position(bytesRead); //LOG.info("Loader Thread Read:"+ bytesRead + " Buffer:" + ++_id); try { _loaderQueue.put(new BufferStruct(buffer, _id)); } catch (InterruptedException e) { LOG.error(CCStringUtils.stringifyException(e)); break; } incomingBuffer = new byte[4096 * 10]; } try { _loaderQueue.put(new BufferStruct(sentinel, ++_id)); } catch (InterruptedException e) { } } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); return; } } }); loaderThread.start(); // set data source ... connection.setDataSource(new DataSource() { int bytesTransferred = 0; @Override public boolean read(NIOBufferList dataBuffer) throws IOException { if (eof.get()) return true; //LOG.info("Connect read callback triggered"); BufferStruct buffer = _loaderQueue.poll(); if (buffer != null) { if (buffer._buffer != sentinel) { //LOG.info("Got Buffer:"+ buffer._id); if (buffer._id == 1) { //LOG.info("Inital Buffer Bytes:" + new String(buffer._buffer.array(),0,10).toString()); } dataBuffer.write(buffer._buffer); bytesTransferred += buffer._buffer.limit(); //LOG.info("Read:" + buffer._buffer.limit() + " Transfered:" + bytesTransferred); return false; } else { //LOG.info("EOF Condition"); dataBuffer.write(sentinel); eof.set(true); return true; } } return false; } }); // open connection connection.open(); // wait for connection to complete ... blockingSemaphore.acquireUninterruptibly(); // kill loader thread loaderThread.interrupt(); try { LOG.info("Waiting for Loader Thread"); loaderThread.join(); LOG.info("Done Waiting for Loader Thread"); } catch (InterruptedException e) { } } finally { is.close(); if (connection != null) { connection.close(); LOG.info("Response Code for File:" + uploadName + "to Host: " + mapping._internalName + " is:" + connection.getResponseHeaders().getHttpResponseCode()); return connection.getResponseHeaders().getHttpResponseCode(); /* if (connection.getResponseHeaders().getHttpResponseCode() != 200) { throw new IOException("Failed to upload file:" + dataFile.getName() + " responseCode:" + connection.getResponseHeaders().getHttpResponseCode()); } */ } } } // something went wrong ??? LOG.error("Failed to upload file:" + uploadName + " unknown response code"); return 500; }