Java tutorial
/* * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved. * * Cloudera, Inc. licenses this file to you under the Apache License, * Version 2.0 (the "License"). You may not use this file except in * compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. See the License for * the specific language governing permissions and limitations under the * License. */ package com.cloudera.recordbreaker.fisheye; import org.apache.wicket.protocol.http.WicketFilter; import org.apache.wicket.protocol.http.WicketServlet; import org.mortbay.jetty.Connector; import org.mortbay.jetty.Server; import org.mortbay.jetty.bio.SocketConnector; import org.mortbay.jetty.servlet.ServletHolder; import org.mortbay.jetty.webapp.WebAppContext; import org.mortbay.resource.ResourceCollection; import com.almworks.sqlite4java.SQLiteException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import java.io.*; import java.util.*; import java.net.URI; import javax.servlet.*; import javax.servlet.http.*; import java.net.URISyntaxException; import com.cloudera.recordbreaker.analyzer.DataQuery; import com.cloudera.recordbreaker.analyzer.FSCrawler; import com.cloudera.recordbreaker.analyzer.FSAnalyzer; import com.cloudera.recordbreaker.analyzer.FileSummary; import com.cloudera.recordbreaker.analyzer.CrawlRuntimeStatus; /*************************************************************** * <code>FishEye</code> is a web app that allows a user to examine * the semantic contents of a filesystem. It makes extensive use of * LearnStructure and other format-parsing code to help the user * figure out the content of files. * * @author "Michael Cafarella" <mjc@cloudera.com> ***************************************************************/ public class FishEye { static final String FISHEYE_METADATA_STORE = "metadata"; static final String FISHEYE_SCHEMA_REPO = "schemarepo"; static FishEye fisheyeInstance; FSAnalyzer analyzer; FSCrawler crawler; AccessController accessCtrl; Date startTime; int fisheyePort; File fisheyeDir; boolean hasTestedQueryServer = false; boolean isQueryServerAvailable = false; public static FishEye getInstance() { return fisheyeInstance; } public FishEye(int port, File fisheyeDir) throws IOException, SQLiteException { if (!fisheyeDir.exists()) { if (!fisheyeDir.mkdir()) { throw new IOException("Cannot create directory: " + fisheyeDir.getCanonicalPath()); } } this.startTime = new Date(System.currentTimeMillis()); this.fisheyeDir = fisheyeDir; this.fisheyePort = port; File fisheyeStore = new File(fisheyeDir, FISHEYE_METADATA_STORE); File fisheyeSchemas = new File(fisheyeDir, FISHEYE_SCHEMA_REPO); this.analyzer = new FSAnalyzer(fisheyeStore, fisheyeSchemas); this.crawler = new FSCrawler(analyzer); this.accessCtrl = new AccessController(); FishEye.fisheyeInstance = this; restartIncompleteCrawl(); } /** * Check if the Hive query server is available */ public boolean isQueryServerAvailable(boolean force) { if (force || (!hasTestedQueryServer)) { hasTestedQueryServer = true; isQueryServerAvailable = DataQuery.getInstance(force).testQueryServer(); } return isQueryServerAvailable; } public boolean restartIncompleteCrawl() { URI fsURI = getFSURI(); if (fsURI != null) { long fsid = analyzer.getCreateFilesystem(fsURI, true); long pendingCrawlId = analyzer.getCreatePendingCrawl(fsid, false); if (pendingCrawlId >= 0) { return crawler.getStartNonblockingCrawl(fsURI); } } return false; } public CrawlRuntimeStatus checkOngoingCrawl() { URI fsUri = getFSURI(); if (fsUri != null) { CrawlRuntimeStatus crs = crawler.isCrawlOngoing(fsUri); return crs; } return null; } public boolean checkCreateCrawl() { URI fsUri = getFSURI(); if (fsUri != null) { return crawler.getStartNonblockingCrawl(fsUri); } return false; } public boolean registerAndCrawlFilesystem(URI fsURI) throws IOException { // REMIND - check the filesystem before proceeding. analyzer.setConfigProperty("fsuri", fsURI.toString()); analyzer.getCreateFilesystem(fsURI, true); return checkCreateCrawl(); } public Date getStartTime() { return startTime; } public int getPort() { return fisheyePort; } public File getFisheyeDir() { return fisheyeDir; } public boolean hasFSAndCrawl() { URI fsURI = getFSURI(); if (fsURI != null) { long fsid = analyzer.getCreateFilesystem(fsURI, false); if (fsid >= 0) { return analyzer.getLatestCompleteCrawl(fsid) >= 0; } } return false; } public URI getFSURI() { try { String uriStr = analyzer.getConfigProperty("fsuri"); if (uriStr == null) { return null; } else { return new URI(uriStr); } } catch (URISyntaxException use) { use.printStackTrace(); return null; } } public void cancelFS() { URI fsUri = getFSURI(); crawler.killOngoingCrawl(fsUri); analyzer.setConfigProperty("fsuri", null); } public String getTopDir() { URI fsUri = getFSURI(); if (fsUri == null) { return null; } long fsid = analyzer.getCreateFilesystem(fsUri, false); if (fsid >= 0) { long crawlid = analyzer.getLatestCompleteCrawl(fsid); if (crawlid >= 0) { Path td = analyzer.getTopDir(crawlid); return td.toString(); } } return null; } public List<FileSummary> getDirParents(String targetDir) { URI fsUri = getFSURI(); if (fsUri == null) { return null; } long fsid = analyzer.getCreateFilesystem(fsUri, false); if (fsid >= 0) { long crawlid = analyzer.getLatestCompleteCrawl(fsid); if (crawlid >= 0) { return analyzer.getDirParents(crawlid, targetDir); } } return null; } public List<FileSummary> getDirChildren(String targetDir) { URI fsUri = getFSURI(); if (fsUri == null) { return null; } long fsid = analyzer.getCreateFilesystem(fsUri, false); if (fsid >= 0) { long crawlid = analyzer.getLatestCompleteCrawl(fsid); if (crawlid >= 0) { return analyzer.getDirChildren(crawlid, targetDir); } } return null; } public FSAnalyzer getAnalyzer() { return analyzer; } public AccessController getAccessController() { return accessCtrl; } public void run() throws Exception { // Jetty object that holds the WicketServlet WicketServlet ws = new WicketServlet(); ServletHolder servletHolder = new ServletHolder(ws); servletHolder.setInitParameter("applicationClassName", "com.cloudera.recordbreaker.fisheye.FishEyeWebApplication"); servletHolder.setInitParameter(WicketFilter.FILTER_MAPPING_PARAM, "/*"); servletHolder.setInitOrder(1); // Config the Jetty WebAppContext object WebAppContext context = new WebAppContext(); context.addServlet(servletHolder, "/*"); String jarDir = this.getClass().getClassLoader().getResource("content/library/bootstrap/1.4.0") .toExternalForm(); String htmlRoot = this.getClass().getClassLoader().getResource("web/fisheye").toExternalForm(); context.setBaseResource(new ResourceCollection(new String[] { htmlRoot, jarDir })); // Start the HTTP server Server server = new Server(); SocketConnector connector = new SocketConnector(); connector.setPort(fisheyePort); server.setConnectors(new Connector[] { connector }); server.setHandler(context); try { server.start(); } catch (Exception ex) { ex.printStackTrace(); } } /** * Start the FishEye Server. */ public static void main(String argv[]) throws Exception { if (argv.length < 2) { System.err.println("Usage: FishEye (-run <port> <fisheyeDir>) (-init <targetdir> [schemadbdir])"); return; } int i = 0; String cmd = argv[i++]; if ("-run".equals(cmd)) { int port = Integer.parseInt(argv[i++]); File fisheyeDir = new File(argv[i++]); FishEye fish = new FishEye(port, fisheyeDir.getCanonicalFile()); fish.run(); } else if ("-init".equals(cmd)) { File targetDir = new File(argv[i++]).getCanonicalFile(); File schemaDbDir = null; if (i < argv.length) { schemaDbDir = new File(argv[i++]).getCanonicalFile(); } if (targetDir.exists()) { throw new IOException("Directory already exists: " + targetDir); } if (!targetDir.mkdirs()) { throw new IOException("Cannot create: " + targetDir); } File metadataStore = new File(targetDir, "metadata"); FSAnalyzer fsa = new FSAnalyzer(metadataStore, schemaDbDir); fsa.close(); } } }