List of usage examples for java.lang String equals
public boolean equals(Object anObject)
From source file:com.peewah.distribuidosfinal.EntryPoint.java
public static void main(String[] args) { //1. Conexin con la base de datos String dbUrl = "jdbc:postgresql://localhost:5432/distribuidosfinal"; try {/*from www . ja v a 2 s . c o m*/ connectionSource = new JdbcConnectionSource(dbUrl); ((JdbcConnectionSource) connectionSource).setUsername("csacanam"); ((JdbcConnectionSource) connectionSource).setPassword("12345678"); } catch (SQLException ex) { System.out.println("Error en la conexin a la base de datos"); } // 2. Data Acces Object (DAO) pattern usuarioDao = null; sistemaOperativoDao = null; maquinaVirtualDao = null; maquinaAppDao = null; appDao = null; cookbookDao = null; cookbookAppDao = null; nodoDao = null; if (connectionSource != null) { try { usuarioDao = DaoManager.createDao(connectionSource, Usuario.class); sistemaOperativoDao = DaoManager.createDao(connectionSource, SistemaOperativo.class); maquinaVirtualDao = DaoManager.createDao(connectionSource, MaquinaVirtual.class); maquinaAppDao = DaoManager.createDao(connectionSource, MaquinaApp.class); appDao = DaoManager.createDao(connectionSource, App.class); cookbookDao = DaoManager.createDao(connectionSource, Cookbook.class); cookbookAppDao = DaoManager.createDao(connectionSource, CookbookApp.class); nodoDao = DaoManager.createDao(connectionSource, Nodo.class); } catch (SQLException ex) { System.out.println("Error en la creacin del DAO"); System.err.println(ex.getMessage()); } } // 3. Crear tabla Usuario si no existe try { TableUtils.createTableIfNotExists(connectionSource, Usuario.class); TableUtils.createTableIfNotExists(connectionSource, SistemaOperativo.class); TableUtils.createTableIfNotExists(connectionSource, MaquinaVirtual.class); TableUtils.createTableIfNotExists(connectionSource, App.class); TableUtils.createTableIfNotExists(connectionSource, MaquinaApp.class); TableUtils.createTableIfNotExists(connectionSource, Cookbook.class); TableUtils.createTableIfNotExists(connectionSource, CookbookApp.class); TableUtils.createTableIfNotExists(connectionSource, Nodo.class); } catch (SQLException ex) { System.out.println("Error creando las tablas"); } //4. Asignacin de puerto ProcessBuilder process = new ProcessBuilder(); Integer puerto; if (process.environment().get("PORT") != null) { puerto = Integer.parseInt(process.environment().get("PORT")); } else { puerto = 8080; } spark.SparkBase.port(puerto); //5. Habilitar Cross-origin resource sharing (CORS) options("/*", (Request rqst, Response rspns) -> { String accessControlRequestHeaders = rqst.headers("Access-Control-Request-Headers"); if (accessControlRequestHeaders != null) { rspns.header("Access-Control-Allow-Headers", accessControlRequestHeaders); } String accessControlRequestMethod = rqst.headers("Access-Control-Request-Method"); if (accessControlRequestMethod != null) { rspns.header("Access-Control-Allow-Methods", accessControlRequestMethod); } return "OK"; }); before((Request rqst, Response rspns) -> { rspns.header("Access-Control-Allow-Origin", "*"); }); after((Request rqst, Response rspns) -> { rspns.type("application/json"); }); //6. Web services //Crear usuario post("/new-user", (Request rqst, Response rspns) -> { //Obtener datos como parmetros String nombre = rqst.queryParams("name"); String username = rqst.queryParams("username"); String password = rqst.queryParams("password"); //Validar si no hay datos vacos if (nombre != null && !nombre.equals("") && username != null && !username.equals("") && password != null && !password.equals("")) { //Crear objeto usuario Usuario usuario = new Usuario(); usuario.setNombre(nombre); usuario.setPassword(password); usuario.setUsername(username); //Crear objeto en base de datos try { usuarioDao.create(usuario); //Crear carpeta File file = new File("/tmp/" + username); if (!file.exists()) { boolean success = file.mkdir(); if (!success) { System.out.println("La carpeta no pudo ser creada"); } } } catch (SQLException ex) { System.out.println("Error creando el usuario"); return false; } } else { System.out.println("No debes dejar campos vacos"); return false; } System.out.println("Usuario creado"); return true; }); //Autenticar usuario post("/auth-user", (Request rqst, Response rspns) -> { //Obtener datos como parmetros String username = rqst.queryParams("username"); String password = rqst.queryParams("password"); //Validar si no hay datos vacos if (username != null && !username.equals("") && password != null && !password.equals("")) { //Validar la dupla usuario-password try { Usuario usuario = usuarioDao.queryForId(username); if (usuario != null && usuario.getPassword().equals(password)) { return true; } } catch (SQLException ex) { } } return false; }); //Listar sistemas operativos disponibles get("/list-so", (Request rqst, Response rspns) -> { List<SistemaOperativo> sistemasOperativos = new ArrayList<>(); try { sistemasOperativos = sistemaOperativoDao.queryForAll(); } catch (SQLException ex) { System.out.println("Error listando los sistemas operativos"); } return sistemasOperativos; }, new JsonTransformer()); //Crear mquina virtual post("/create-machine", (Request rqst, Response rspns) -> { try { //Obtener parmetros String username = rqst.queryParams("username"); String nombreMaquina = rqst.queryParams("nombreMaquina"); String nombreSO = rqst.queryParams("nombreSO"); Usuario user = usuarioDao.queryForId(username); SistemaOperativo so = sistemaOperativoDao.queryForId(nombreSO); if (user != null && so != null) { //Crear mquina virtual MaquinaVirtual maquinaVirtual = new MaquinaVirtual(); maquinaVirtual.setNombre(nombreMaquina); maquinaVirtual.setSistemaOperativo(sistemaOperativoDao.queryForId(nombreSO)); maquinaVirtual.setUsername(usuarioDao.queryForId(username)); maquinaVirtualDao.create(maquinaVirtual); //Crear carpeta String path = "/tmp/" + username + "/" + nombreMaquina; File file = new File(path); if (!file.exists()) { boolean success = file.mkdir(); if (!success) { System.out.println("No se pudo crear la carpeta para la mquina"); } else { //Crear Vagrantfile try (Writer writer = new BufferedWriter( new OutputStreamWriter(new FileOutputStream(path + "/Vagrantfile"), "UTF-8"))) { writer.write("VAGRANTFILE_API_VERSION = \"2\"\n"); writer.write("Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n"); writer.write("\n"); writer.write("end\n"); } } } return true; } else { return false; } } catch (SQLException ex) { System.out.println("Error creando la mquina virtual"); return false; } }); //Eliminar usuario y sus mquinas virtuales asociadas post("/delete-user", (Request rqst, Response rspns) -> { String userLogged = rqst.queryParams("usernameLogged"); String nombreUsuario = rqst.queryParams("usernameToDelete"); if (userLogged != null && !userLogged.equals("") && nombreUsuario != null && !userLogged.equals("") && userLogged.equals("admin")) { try { Usuario user = usuarioDao.queryForId(nombreUsuario); if (user != null) { //Eliminar mquinas virtuales del usuario Collection<MaquinaVirtual> maquinasUsuario = user.getMaquinasVirtuales(); for (MaquinaVirtual maquina : maquinasUsuario) { //Eliminar apps de las mquinas virtuales del usuario QueryBuilder<MaquinaApp, String> queryBuilder = maquinaAppDao.queryBuilder(); queryBuilder.where().eq(MaquinaApp.MACHINE_FIELD, maquina.getId()); PreparedQuery<MaquinaApp> preparedQuery = queryBuilder.prepare(); Collection<MaquinaApp> maquinasApps = maquinaAppDao.query(preparedQuery); maquinaAppDao.delete(maquinasApps); //Eliminar la mquina virtual maquinaVirtualDao.delete(maquina); } //Eliminar usuario usuarioDao.delete(user); //Eliminar carpeta del usuario FileUtils.deleteDirectory(new File("/tmp/" + nombreUsuario)); return true; } else { return false; } } catch (SQLException ex) { System.out.println("Error eliminando el usuario"); return false; } } else { System.out.println("No tiene permisos para realizar esta accin"); return false; } }); //Listar mquinas virtuales de un usuario get("/list-machines", (Request rqst, Response rspns) -> { String username = rqst.queryParams("username"); return listMachines(username); }, new JsonTransformer()); //Listar usuarios get("/list-users", (Request rqst, Response rspns) -> { String username = rqst.queryParams("usernameLogged"); if (username.equals("admin")) { List<Usuario> usuarios = new ArrayList<>(); try { usuarios = usuarioDao.queryForAll(); } catch (SQLException ex) { System.out.println("Error listando los usuarios"); } return usuarios; } else { System.out.println("No tiene permisos para realizar esta accin"); return "No tiene permisos para realizar esta accin"; } }, new JsonTransformer()); // Agregar nodo a una mquina virtual post("/add-node", (Request rqst, Response rspns) -> { String nombreNodo = rqst.queryParams("nombreNodo"); String ipPrivada = rqst.queryParams("ipPrivada"); String ipPublica = rqst.queryParams("ipPublica"); String mascaraRed = rqst.queryParams("mascaraRed"); String cantidadMemoria = rqst.queryParams("cantidadMemoria"); String cantidadCPU = rqst.queryParams("cantidadCPU"); String interfazPuente = rqst.queryParams("interfazPuente"); String parametrosJSON = rqst.queryParams("parametrosJSON"); String nombreMaquina = rqst.queryParams("nombreMaquina"); String userLogged = rqst.queryParams("userLogged"); Usuario user = usuarioDao.queryForId(userLogged); if (user != null) { //Buscar mquina QueryBuilder<MaquinaVirtual, Integer> queryBuilder = maquinaVirtualDao.queryBuilder(); queryBuilder.where().eq(MaquinaVirtual.USERNAME_FIELD, userLogged); queryBuilder.where().eq(MaquinaVirtual.NOMBRE_FIELD, nombreMaquina); PreparedQuery<MaquinaVirtual> preparedQuery = queryBuilder.prepare(); List<MaquinaVirtual> maquinasUser = maquinaVirtualDao.query(preparedQuery); //Existe la mquina if (maquinasUser.size() > 0 && maquinasUser.get(0).getNombre().equals(nombreMaquina)) { //Crear nodo Nodo nodo = new Nodo(); nodo.setNombre(nombreNodo); nodo.setCantidadCPU(cantidadCPU); nodo.setCantidadMemoria(cantidadMemoria); nodo.setInterfazPuente(interfazPuente); nodo.setIpPrivada(ipPrivada); nodo.setIpPublica(ipPublica); nodo.setMascaraRed(mascaraRed); nodo.setParametrosJSON(parametrosJSON); nodo.setMaquinaVirtual(maquinasUser.get(0)); nodoDao.create(nodo); //Crear nodo en Vagrantfile insertarNodoEnVagrantFile("/tmp/" + userLogged + "/" + nombreMaquina, nodo); return true; } } return false; }); //Listar apps para un SO get("/list-apps", (Request rqst, Response rspns) -> { String nameSO = rqst.queryParams("nombreSO"); SistemaOperativo buscado = sistemaOperativoDao.queryForId(nameSO); if (buscado != null) { QueryBuilder<App, String> queryBuilder = appDao.queryBuilder(); queryBuilder.where().eq(App.SO_FIELD, buscado.getNombre()); PreparedQuery<App> preparedQuery = queryBuilder.prepare(); Collection<App> aplicaciones = appDao.query(preparedQuery); return aplicaciones; } else { return "El SO buscado no existe"; } }, new JsonTransformer()); //Listar apps para una maquina virtual get("/list-app-installed", (Request rqst, Response rspns) -> { String userLogged = rqst.queryParams("userLogged"); String machineName = rqst.queryParams("nombreMaquina"); List<MaquinaApp> maquinaApp; List<App> apps = new ArrayList<>(); //Buscar si el usuario loggeado existe Usuario buscado; MaquinaVirtual buscada = null; try { buscado = usuarioDao.queryForId(userLogged); if (buscado != null) { //Lista de maquinas virtuales del usuario Collection<MaquinaVirtual> maquinasVirtuales = listMachines(userLogged); //Revisar si la maquina virtual buscada pertenece al usuario loggeado for (MaquinaVirtual maquina : maquinasVirtuales) { if (maquina.getNombre().equals(machineName)) { buscada = maquina; break; } } if (buscada != null) { //Obtener la lista de aplicaciones de la maquina virtual QueryBuilder<MaquinaApp, String> queryBuilderN = maquinaAppDao.queryBuilder(); queryBuilderN.where().eq(MaquinaApp.MACHINE_FIELD, buscada.getId()); PreparedQuery<MaquinaApp> preparedQueryN = queryBuilderN.prepare(); maquinaApp = maquinaAppDao.query(preparedQueryN); if (maquinaApp.size() > 0) { for (MaquinaApp m : maquinaApp) { apps.add(m.getApp()); } } } else { System.out.println("La maquina no existe para el usuario buscado"); } } else { System.out.println("El usuario loggeado no existe"); } } catch (SQLException ex) { System.out.println("Error listando las apps instaladas"); } return apps; }, new JsonTransformer()); //Listar nodos de una maquina virtual get("/list-node", (Request rqst, Response rspns) -> { String userLogged = rqst.queryParams("userLogged"); String nombreMaquina = rqst.queryParams("nombreMaquina"); //Inicializar la lista de nodos que se va a retornar Collection<Nodo> nodos = new ArrayList<>(); //Validar que no hayan campos vacios if (userLogged != null && !userLogged.equals("") && nombreMaquina != null && !nombreMaquina.equals("")) { //Buscar el usuario loggeado Usuario user = usuarioDao.queryForId(userLogged); //Verificar que el usuario existe if (user != null) { //Obtener las maquinas virtuales del usuario List<MaquinaVirtual> maquinasVirtuales = listMachines(userLogged); for (MaquinaVirtual m : maquinasVirtuales) { if (m.getNombre().equals(nombreMaquina)) { nodos = m.getNodos(); } } } else { System.out.println("El usuario loggeado no existe"); } } else { System.out.println("No pueden haber parametros vacios"); } return nodos; }, new JsonTransformer()); //Eliminar maquina virtual y aplicaciones asociadas post("/delete-vm", (Request rqst, Response rspns) -> { String usernameLogged = rqst.queryParams("usernameLogged"); String nombreMaquina = rqst.queryParams("nombreMaquina"); MaquinaVirtual buscada = null; List<MaquinaApp> maquinaApp; //Verificar que los parametros recibidos no son null if (usernameLogged != null && !usernameLogged.equals("") && nombreMaquina != null && !nombreMaquina.equals("")) { Usuario user = usuarioDao.queryForId(usernameLogged); if (user != null) { //Obtener las maquinas virtuales del usuario List<MaquinaVirtual> maquinasVirtuales = listMachines(usernameLogged); //Buscar la maquina virtual a eliminar dentro de las maquinas del usuario for (MaquinaVirtual m : maquinasVirtuales) { if (m.getNombre().equals(nombreMaquina)) { buscada = m; break; } } //Verificar que la maquina buscada pertenece al usuario en cuestion if (buscada != null) { //Obtener la lista de aplicaciones de la maquina virtual QueryBuilder<MaquinaApp, String> queryBuilder = maquinaAppDao.queryBuilder(); queryBuilder.where().eq(MaquinaApp.MACHINE_FIELD, buscada.getId()); PreparedQuery<MaquinaApp> preparedQuery = queryBuilder.prepare(); maquinaApp = maquinaAppDao.query(preparedQuery); if (maquinaApp.size() > 0) { //Eliminar las aplicaciones for (MaquinaApp i : maquinaApp) { maquinaAppDao.delete(i); } } else { System.out.println("No existen aplicaciones para la maquina virtual en cuestion"); } //Eliminar mquina virtual maquinaVirtualDao.delete(buscada); //Eliminar carpeta de la maquina virtual FileUtils.deleteDirectory(new File("/tmp/" + usernameLogged + "/" + nombreMaquina)); return true; } } else { System.out.println("EL usuario loggeado no existe"); } } else { System.out.println("No pueden haber campos vacios"); } return false; }); //Correr mquina virtual post("/run-vm", (Request rqst, Response rspns) -> { String username = rqst.queryParams("userLogged"); String nombreMaquina = rqst.queryParams("nombreMaquina"); MaquinaVirtual maquinaBuscada = null; if (username != null && nombreMaquina != null && !nombreMaquina.equals("") && !nombreMaquina.equals("")) { Usuario user = usuarioDao.queryForId(username); if (user != null) { //Listar mquinas virtuales del usuario List<MaquinaVirtual> maquinasVirtuales = listMachines(username); for (MaquinaVirtual maquina : maquinasVirtuales) { if (maquina.getNombre().equals(nombreMaquina)) { maquinaBuscada = maquina; break; } } if (maquinaBuscada != null) { try { //Comando para ejecutar el comando vagrant up en el shell ProcessBuilder pb = new ProcessBuilder("vagrant up"); Process p; String path = "/tmp/" + username + "/" + nombreMaquina; File file = new File(path); //Validar si es un directorio if (file.exists() && file.isDirectory()) { pb.directory(file); p = pb.start(); return true; } } catch (IOException ex) { } } } } return false; }); //Destruir mquina virtual post("/destroy-vm", (Request rqst, Response rspns) -> { String username = rqst.queryParams("userLogged"); String nombreMaquina = rqst.queryParams("nombreMaquina"); MaquinaVirtual maquinaBuscada = null; if (username != null && nombreMaquina != null && !nombreMaquina.equals("") && !nombreMaquina.equals("")) { Usuario user = usuarioDao.queryForId(username); if (user != null) { //Listar mquinas virtuales del usuario List<MaquinaVirtual> maquinasVirtuales = listMachines(username); for (MaquinaVirtual maquina : maquinasVirtuales) { if (maquina.getNombre().equals(nombreMaquina)) { maquinaBuscada = maquina; break; } } if (maquinaBuscada != null) { try { //Comando para ejecutar el comando vagrant up en el shell ProcessBuilder pb = new ProcessBuilder("vagrant destroy -f"); Process p; String path = "/tmp/" + username + "/" + nombreMaquina; File file = new File(path); //Validar si es un directorio if (file.exists() && file.isDirectory()) { pb.directory(file); p = pb.start(); return true; } } catch (IOException ex) { } } } } return false; }); //Reanudar mquina virtual post("/resume-vm", (Request rqst, Response rspns) -> { String username = rqst.queryParams("userLogged"); String nombreMaquina = rqst.queryParams("nombreMaquina"); MaquinaVirtual maquinaBuscada = null; if (username != null && nombreMaquina != null && !nombreMaquina.equals("") && !nombreMaquina.equals("")) { Usuario user = usuarioDao.queryForId(username); if (user != null) { //Listar mquinas virtuales del usuario List<MaquinaVirtual> maquinasVirtuales = listMachines(username); for (MaquinaVirtual maquina : maquinasVirtuales) { if (maquina.getNombre().equals(nombreMaquina)) { maquinaBuscada = maquina; break; } } if (maquinaBuscada != null) { try { //Comando para ejecutar el comando vagrant up en el shell ProcessBuilder pb = new ProcessBuilder("vagrant resume"); Process p; String path = "/tmp/" + username + "/" + nombreMaquina; File file = new File(path); //Validar si es un directorio if (file.exists() && file.isDirectory()) { pb.directory(file); p = pb.start(); return true; } } catch (IOException ex) { } } } } return false; }); //Asociar app a una mquina virtual post("/associate-app", (Request rqst, Response rspns) -> { String username = rqst.queryParams("username"); String nombreMaquina = rqst.queryParams("nombreMaquina"); String nombreApp = rqst.queryParams("nombreApp"); Usuario user = usuarioDao.queryForId(username); if (user != null) { //Verificar si el usuario tiene la mquina List<MaquinaVirtual> maquinas = listMachines(username); MaquinaVirtual buscada = null; for (MaquinaVirtual maquina : maquinas) { if (maquina.getNombre().equals(nombreMaquina)) { buscada = maquina; break; } } if (buscada != null) { App app = appDao.queryForId(nombreApp); //Verificar si la app existe y si est para el mismo sistema operativo que tiene la mquina if (app != null && app.getSistemaOperativo().getNombre() .equals(buscada.getSistemaOperativo().getNombre())) { //Agregar a la base de datos MaquinaApp maquinaApp = new MaquinaApp(buscada, app); maquinaAppDao.create(maquinaApp); //Crear registro en el Vagrantfile String path = "/tmp/" + username + "/" + nombreMaquina; insertarCookbooksANodos(new ArrayList(buscada.getNodos()), app, path); return true; } else { System.out.println("La app no existe"); } } else { System.out.println("No se encontr la mquina en la lista del usuario"); } } else { System.out.println("El usuario no existe"); } return false; }); //Listar todas las aplicaciones get("/list-apps-all", (Request rqst, Response rspns) -> { return appDao.queryForAll(); }, new JsonTransformer()); // Cargar datos de prueba get("/add-testdata", (Request rqst, Response rspns) -> { try { if (connectionSource != null) { TableUtils.createTableIfNotExists(connectionSource, Usuario.class); TableUtils.createTableIfNotExists(connectionSource, SistemaOperativo.class); TableUtils.createTableIfNotExists(connectionSource, MaquinaVirtual.class); TableUtils.createTableIfNotExists(connectionSource, App.class); TableUtils.createTableIfNotExists(connectionSource, MaquinaApp.class); TableUtils.createTableIfNotExists(connectionSource, Cookbook.class); TableUtils.createTableIfNotExists(connectionSource, CookbookApp.class); TableUtils.createTableIfNotExists(connectionSource, Nodo.class); } testData(); } catch (SQLException ex) { return "Error agregando informacin de prueba"; } return "OK"; }); //Eliminar datos de prueba get("/delete-testdata", (Request rqst, Response rspns) -> { try { TableUtils.dropTable(connectionSource, Usuario.class, true); TableUtils.dropTable(connectionSource, MaquinaVirtual.class, true); TableUtils.dropTable(connectionSource, SistemaOperativo.class, true); TableUtils.dropTable(connectionSource, App.class, true); TableUtils.dropTable(connectionSource, Cookbook.class, true); TableUtils.dropTable(connectionSource, MaquinaApp.class, true); TableUtils.dropTable(connectionSource, CookbookApp.class, true); TableUtils.dropTable(connectionSource, Nodo.class, true); } catch (SQLException ex) { return "Error eliminando la informacin"; } return "OK"; }); }
From source file:com.netthreads.mavenize.Mavenize.java
/** * Main method./* w w w . ja v a 2 s.c o m*/ * * @param args */ public static void main(String[] args) { if (args.length > 1) { String sourcePath = ""; String targetPath = ""; String projectTypeName = ProjectType.Types.DEFAULT.toString(); String version = PomGenerator.DEFAULT_VERSION; String packaging = PomGenerator.PACKAGE_TYPES[0]; boolean isInput = false; boolean isOutput = false; for (String arg : args) { try { if (arg.startsWith(ARG_INPUT)) { sourcePath = arg.substring(ARG_INPUT.length()); isInput = true; } else if (arg.startsWith(ARG_OUTPUT)) { targetPath = arg.substring(ARG_OUTPUT.length()); isOutput = true; } else if (arg.startsWith(ARG_TYPE)) { projectTypeName = arg.substring(ARG_TYPE.length()); } else if (arg.startsWith(ARG_VERSION)) { version = arg.substring(ARG_VERSION.length()); } else if (arg.startsWith(ARG_PACKAGING)) { packaging = arg.substring(ARG_PACKAGING.length()); } } catch (Exception e) { logger.error("Can't process argument, " + arg + ", " + e.getMessage()); } } // Project type. ProjectType projectType = ProjectTypeFactory.instance().getProjectType(projectTypeName); // Execute conversion. try { if (isInput && isOutput) { if (!sourcePath.equals(targetPath)) { Mavenize mvnGather = new Mavenize(null); mvnGather.process(sourcePath, targetPath, projectType, version, packaging); } else { throw new MavenizeException("Input and output directories cannot be the same."); } } else { throw new MavenizeException("You must specify input and output directories"); } } catch (MavenizeException e) { logger.error("Application error, " + e); } catch (IOException ioe) { logger.error("Application error, " + ioe); } } else { System.out.println(APP_MESSAGE + ARGS_MESSAGE); } }
From source file:com.aerospike.load.AerospikeLoad.java
public static void main(String[] args) throws IOException { Thread statPrinter = new Thread(new PrintStat(counters)); try {/*from w w w.j av a 2 s. co m*/ log.info("Aerospike loader started"); Options options = new Options(); options.addOption("h", "host", true, "Server hostname (default: localhost)"); options.addOption("p", "port", true, "Server port (default: 3000)"); options.addOption("n", "namespace", true, "Namespace (default: test)"); options.addOption("s", "set", true, "Set name. (default: null)"); options.addOption("c", "config", true, "Column definition file name"); options.addOption("wt", "write-threads", true, "Number of writer threads (default: Number of cores * 5)"); options.addOption("rt", "read-threads", true, "Number of reader threads (default: Number of cores * 1)"); options.addOption("l", "rw-throttle", true, "Throttling of reader to writer(default: 10k) "); options.addOption("tt", "transaction-timeout", true, "write transaction timeout in miliseconds(default: No timeout)"); options.addOption("et", "expiration-time", true, "Expiration time of records in seconds (default: never expire)"); options.addOption("T", "timezone", true, "Timezone of source where data dump is taken (default: local timezone)"); options.addOption("ec", "abort-error-count", true, "Error count to abort (default: 0)"); options.addOption("wa", "write-action", true, "Write action if key already exists (default: update)"); options.addOption("v", "verbose", false, "Logging all"); options.addOption("u", "usage", false, "Print usage."); CommandLineParser parser = new PosixParser(); CommandLine cl = parser.parse(options, args, false); if (args.length == 0 || cl.hasOption("u")) { logUsage(options); return; } if (cl.hasOption("l")) { rwThrottle = Integer.parseInt(cl.getOptionValue("l")); } else { rwThrottle = Constants.READLOAD; } // Get all command line options params = Utils.parseParameters(cl); //Get client instance AerospikeClient client = new AerospikeClient(params.host, params.port); if (!client.isConnected()) { log.error("Client is not able to connect:" + params.host + ":" + params.port); return; } if (params.verbose) { log.setLevel(Level.DEBUG); } // Get available processors to calculate default number of threads int cpus = Runtime.getRuntime().availableProcessors(); nWriterThreads = cpus * scaleFactor; nReaderThreads = cpus; // Get writer thread count if (cl.hasOption("wt")) { nWriterThreads = Integer.parseInt(cl.getOptionValue("wt")); nWriterThreads = (nWriterThreads > 0 ? (nWriterThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nWriterThreads) : 1); log.debug("Using writer Threads: " + nWriterThreads); } writerPool = Executors.newFixedThreadPool(nWriterThreads); // Get reader thread count if (cl.hasOption("rt")) { nReaderThreads = Integer.parseInt(cl.getOptionValue("rt")); nReaderThreads = (nReaderThreads > 0 ? (nReaderThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nReaderThreads) : 1); log.debug("Using reader Threads: " + nReaderThreads); } String columnDefinitionFileName = cl.getOptionValue("c", null); log.debug("Column definition files/directory: " + columnDefinitionFileName); if (columnDefinitionFileName == null) { log.error("Column definition files/directory not specified. use -c <file name>"); return; } File columnDefinitionFile = new File(columnDefinitionFileName); if (!columnDefinitionFile.exists()) { log.error("Column definition files/directory does not exist: " + Utils.getFileName(columnDefinitionFileName)); return; } // Get data file list String[] files = cl.getArgs(); if (files.length == 0) { log.error("No data file Specified: add <file/dir name> to end of the command "); return; } List<String> allFileNames = new ArrayList<String>(); allFileNames = Utils.getFileNames(files); if (allFileNames.size() == 0) { log.error("Given datafiles/directory does not exist"); return; } for (int i = 0; i < allFileNames.size(); i++) { log.debug("File names:" + Utils.getFileName(allFileNames.get(i))); File file = new File(allFileNames.get(i)); counters.write.recordTotal = counters.write.recordTotal + file.length(); } //remove column definition file from list allFileNames.remove(columnDefinitionFileName); log.info("Number of data files:" + allFileNames.size()); /** * Process column definition file to get meta data and bin mapping. */ metadataColumnDefs = new ArrayList<ColumnDefinition>(); binColumnDefs = new ArrayList<ColumnDefinition>(); metadataConfigs = new HashMap<String, String>(); if (Parser.processJSONColumnDefinitions(columnDefinitionFile, metadataConfigs, metadataColumnDefs, binColumnDefs, params)) { log.info("Config file processed."); } else { throw new Exception("Config file parsing Error"); } // Add metadata of config to parameters String metadata; if ((metadata = metadataConfigs.get(Constants.INPUT_TYPE)) != null) { params.fileType = metadata; if (params.fileType.equals(Constants.CSV_FILE)) { // Version check metadata = metadataConfigs.get(Constants.VERSION); String[] vNumber = metadata.split("\\."); int v1 = Integer.parseInt(vNumber[0]); int v2 = Integer.parseInt(vNumber[1]); if ((v1 <= Constants.MajorV) && (v2 <= Constants.MinorV)) { log.debug("Config version used:" + metadata); } else throw new Exception("\"" + Constants.VERSION + ":" + metadata + "\" is not Supported"); // Set delimiter if ((metadata = metadataConfigs.get(Constants.DELIMITER)) != null && metadata.length() == 1) { params.delimiter = metadata.charAt(0); } else { log.warn("\"" + Constants.DELIMITER + ":" + metadata + "\" is not properly specified in config file. Default is ','"); } if ((metadata = metadataConfigs.get(Constants.IGNORE_FIRST_LINE)) != null) { params.ignoreFirstLine = metadata.equals("true"); } else { log.warn("\"" + Constants.IGNORE_FIRST_LINE + ":" + metadata + "\" is not properly specified in config file. Default is false"); } if ((metadata = metadataConfigs.get(Constants.COLUMNS)) != null) { counters.write.colTotal = Integer.parseInt(metadata); } else { throw new Exception("\"" + Constants.COLUMNS + ":" + metadata + "\" is not properly specified in config file"); } } else { throw new Exception("\"" + params.fileType + "\" is not supported in config file"); } } else { throw new Exception("\"" + Constants.INPUT_TYPE + "\" is not specified in config file"); } // add config input to column definitions if (params.fileType.equals(Constants.CSV_FILE)) { List<String> binName = null; if (params.ignoreFirstLine) { String line; BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(allFileNames.get(0)), "UTF8")); if ((line = br.readLine()) != null) { binName = Parser.getCSVRawColumns(line, params.delimiter); br.close(); if (binName.size() != counters.write.colTotal) { throw new Exception("Number of column in config file and datafile are mismatch." + " Datafile: " + Utils.getFileName(allFileNames.get(0)) + " Configfile: " + Utils.getFileName(columnDefinitionFileName)); } } } //update columndefs for metadata for (int i = 0; i < metadataColumnDefs.size(); i++) { if (metadataColumnDefs.get(i).staticValue) { } else { if (metadataColumnDefs.get(i).binValuePos < 0) { if (metadataColumnDefs.get(i).columnName == null) { if (metadataColumnDefs.get(i).jsonPath == null) { log.error("dynamic metadata having improper info" + metadataColumnDefs.toString()); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(metadataColumnDefs.get(i).binValueHeader) != -1) { metadataColumnDefs.get(i).binValuePos = binName .indexOf(metadataColumnDefs.get(i).binValueHeader); } else { throw new Exception("binName missing in data file:" + metadataColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) metadataColumnDefs.get(i).binValueHeader = binName .get(metadataColumnDefs.get(i).binValuePos); } } if ((!metadataColumnDefs.get(i).staticValue) && (metadataColumnDefs.get(i).binValuePos < 0)) { throw new Exception("Information for bin mapping is missing in config file:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader == Constants.SET && !metadataColumnDefs.get(i).srcType.equals(SrcColumnType.STRING)) { throw new Exception("Set name should be string type:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader.equalsIgnoreCase(Constants.SET) && params.set != null) { throw new Exception( "Set name is given both in config file and commandline. Provide only once."); } } //update columndefs for bins for (int i = 0; i < binColumnDefs.size(); i++) { if (binColumnDefs.get(i).staticName) { } else { if (binColumnDefs.get(i).binNamePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(binColumnDefs.get(i).binNameHeader) != -1) { binColumnDefs.get(i).binNamePos = binName .indexOf(binColumnDefs.get(i).binNameHeader); } else { throw new Exception("binName missing in data file:" + binColumnDefs.get(i).binNameHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binNameHeader = binName.get(binColumnDefs.get(i).binNamePos); } } if (binColumnDefs.get(i).staticValue) { } else { if (binColumnDefs.get(i).binValuePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.contains(binColumnDefs.get(i).binValueHeader)) { binColumnDefs.get(i).binValuePos = binName .indexOf(binColumnDefs.get(i).binValueHeader); } else if (!binColumnDefs.get(i).binValueHeader.toLowerCase() .equals(Constants.SYSTEM_TIME)) { throw new Exception("Wrong column name mentioned in config file:" + binColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binValueHeader = binName.get(binColumnDefs.get(i).binValuePos); } //check for missing entries in config file if (binColumnDefs.get(i).binValuePos < 0 && binColumnDefs.get(i).binValueHeader == null) { throw new Exception("Information missing(Value header or bin mapping) in config file:" + binColumnDefs.get(i)); } //check for proper data type in config file. if (binColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + binColumnDefs.get(i)); } //check for valid destination type if ((binColumnDefs.get(i).srcType.equals(SrcColumnType.TIMESTAMP) || binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB)) && binColumnDefs.get(i).dstType == null) { throw new Exception("Destination type is not mentioned: " + binColumnDefs.get(i)); } //check for encoding if (binColumnDefs.get(i).dstType != null && binColumnDefs.get(i).encoding == null) { throw new Exception( "Encoding is not given for src-dst type conversion:" + binColumnDefs.get(i)); } //check for valid encoding if (binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB) && !binColumnDefs.get(i).encoding.equals(Constants.HEX_ENCODING)) { throw new Exception("Wrong encoding for blob data:" + binColumnDefs.get(i)); } } //Check static bin name mapped to dynamic bin value if ((binColumnDefs.get(i).binNamePos == binColumnDefs.get(i).binValuePos) && (binColumnDefs.get(i).binNamePos != -1)) { throw new Exception("Static bin name mapped to dynamic bin value:" + binColumnDefs.get(i)); } //check for missing entries in config file if (binColumnDefs.get(i).binNameHeader == null && binColumnDefs.get(i).binNameHeader.length() > Constants.BIN_NAME_LENGTH) { throw new Exception("Information missing binName or large binName in config file:" + binColumnDefs.get(i)); } } } log.info(params.toString()); log.debug("MetadataConfig:" + metadataColumnDefs); log.debug("BinColumnDefs:" + binColumnDefs); // Start PrintStat thread statPrinter.start(); // Reader pool size ExecutorService readerPool = Executors.newFixedThreadPool( nReaderThreads > allFileNames.size() ? allFileNames.size() : nReaderThreads); log.info("Reader pool size : " + nReaderThreads); // Submit all tasks to writer threadpool. for (String aFile : allFileNames) { log.debug("Submitting task for: " + aFile); readerPool.submit(new AerospikeLoad(aFile, client, params)); } // Wait for reader pool to complete readerPool.shutdown(); log.info("Shutdown down reader thread pool"); while (!readerPool.isTerminated()) ; //readerPool.awaitTermination(20, TimeUnit.MINUTES); log.info("Reader thread pool terminated"); // Wait for writer pool to complete after getting all tasks from reader pool writerPool.shutdown(); log.info("Shutdown down writer thread pool"); while (!writerPool.isTerminated()) ; log.info("Writer thread pool terminated"); // Print final statistic of aerospike-loader. log.info("Final Statistics of importer: (Succesfull Writes = " + counters.write.writeCount.get() + ", " + "Errors=" + (counters.write.writeErrors.get() + counters.write.readErrors.get() + counters.write.processingErrors.get()) + "(" + (counters.write.writeErrors.get()) + "-Write," + counters.write.readErrors.get() + "-Read," + counters.write.processingErrors.get() + "-Processing)"); } catch (Exception e) { log.error(e); if (log.isDebugEnabled()) { e.printStackTrace(); } } finally { // Stop statistic printer thread. statPrinter.interrupt(); log.info("Aerospike loader completed"); } }
From source file:gov.nasa.jpl.memex.pooledtimeseries.PoT.java
public static void main(String[] args) { System.loadLibrary(Core.NATIVE_LIBRARY_NAME); Option fileOpt = OptionBuilder.withArgName("file").hasArg().withLongOpt("file") .withDescription("Path to a single file").create('f'); Option dirOpt = OptionBuilder.withArgName("directory").hasArg().withLongOpt("dir") .withDescription("A directory with image files in it").create('d'); Option helpOpt = OptionBuilder.withLongOpt("help").withDescription("Print this message.").create('h'); Option pathFileOpt = OptionBuilder.withArgName("path file").hasArg().withLongOpt("pathfile") .withDescription(//from w ww . j a v a 2 s. c o m "A file containing full absolute paths to videos. Previous default was memex-index_temp.txt") .create('p'); Option outputFileOpt = OptionBuilder.withArgName("output file").withLongOpt("outputfile").hasArg() .withDescription("File containing similarity results. Defaults to ./similarity.txt").create('o'); Option jsonOutputFlag = OptionBuilder.withArgName("json output").withLongOpt("json") .withDescription("Set similarity output format to JSON. Defaults to .txt").create('j'); Option similarityFromFeatureVectorsOpt = OptionBuilder .withArgName("similarity from FeatureVectors directory") .withLongOpt("similarityFromFeatureVectorsDirectory").hasArg() .withDescription("calculate similarity matrix from given directory of feature vectors").create('s'); Options options = new Options(); options.addOption(dirOpt); options.addOption(pathFileOpt); options.addOption(fileOpt); options.addOption(helpOpt); options.addOption(outputFileOpt); options.addOption(jsonOutputFlag); options.addOption(similarityFromFeatureVectorsOpt); // create the parser CommandLineParser parser = new GnuParser(); try { // parse the command line arguments CommandLine line = parser.parse(options, args); String directoryPath = null; String pathFile = null; String singleFilePath = null; String similarityFromFeatureVectorsDirectory = null; ArrayList<Path> videoFiles = null; if (line.hasOption("dir")) { directoryPath = line.getOptionValue("dir"); } if (line.hasOption("pathfile")) { pathFile = line.getOptionValue("pathfile"); } if (line.hasOption("file")) { singleFilePath = line.getOptionValue("file"); } if (line.hasOption("outputfile")) { outputFile = line.getOptionValue("outputfile"); } if (line.hasOption("json")) { outputFormat = OUTPUT_FORMATS.JSON; } if (line.hasOption("similarityFromFeatureVectorsDirectory")) { similarityFromFeatureVectorsDirectory = line .getOptionValue("similarityFromFeatureVectorsDirectory"); } if (line.hasOption("help") || (line.getOptions() == null || (line.getOptions() != null && line.getOptions().length == 0)) || (directoryPath != null && pathFile != null && !directoryPath.equals("") && !pathFile.equals(""))) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("pooled_time_series", options); System.exit(1); } if (directoryPath != null) { File dir = new File(directoryPath); List<File> files = (List<File>) FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE); videoFiles = new ArrayList<Path>(files.size()); for (File file : files) { String filePath = file.toString(); // When given a directory to load videos from we need to ensure that we // don't try to load the of.txt and hog.txt intermediate result files // that results from previous processing runs. if (!filePath.contains(".txt")) { videoFiles.add(file.toPath()); } } LOG.info("Added " + videoFiles.size() + " video files from " + directoryPath); } if (pathFile != null) { Path list_file = Paths.get(pathFile); videoFiles = loadFiles(list_file); LOG.info("Loaded " + videoFiles.size() + " video files from " + pathFile); } if (singleFilePath != null) { Path singleFile = Paths.get(singleFilePath); LOG.info("Loaded file: " + singleFile); videoFiles = new ArrayList<Path>(1); videoFiles.add(singleFile); } if (similarityFromFeatureVectorsDirectory != null) { File dir = new File(similarityFromFeatureVectorsDirectory); List<File> files = (List<File>) FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE); videoFiles = new ArrayList<Path>(files.size()); for (File file : files) { String filePath = file.toString(); // We need to load only the *.of.txt and *.hog.txt values if (filePath.endsWith(".of.txt")) { videoFiles.add(file.toPath()); } if (filePath.endsWith(".hog.txt")) { videoFiles.add(file.toPath()); } } LOG.info("Added " + videoFiles.size() + " feature vectors from " + similarityFromFeatureVectorsDirectory); evaluateSimilarity(videoFiles, 1); } else { evaluateSimilarity(videoFiles, 1); } LOG.info("done."); } catch (ParseException exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); } }
From source file:com.netscape.cms.servlet.test.CATest.java
public static void main(String args[]) { String host = null;//from w w w . j a va 2 s . com String port = null; String token_pwd = null; String db_dir = "./"; String protocol = "http"; // parse command line arguments Options options = new Options(); options.addOption("h", true, "Hostname of the CA"); options.addOption("p", true, "Port of the CA"); options.addOption("s", true, "Attempt Optional Secure SSL connection"); options.addOption("w", true, "Token password"); options.addOption("d", true, "Directory for tokendb"); options.addOption("c", true, "Optional SSL Client cert Nickname"); try { CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("h")) { host = cmd.getOptionValue("h"); } else { System.err.println("Error: no hostname provided."); usage(options); } if (cmd.hasOption("p")) { port = cmd.getOptionValue("p"); } else { System.err.println("Error: no port provided"); usage(options); } if (cmd.hasOption("w")) { token_pwd = cmd.getOptionValue("w"); } else { log("Notice: no token password provided"); } if (cmd.hasOption("d")) { db_dir = cmd.getOptionValue("d"); } if (cmd.hasOption("s")) { if (cmd.getOptionValue("s") != null && cmd.getOptionValue("s").equals("true")) { protocol = "https"; } } if (cmd.hasOption("c")) { String nick = cmd.getOptionValue("c"); if (nick != null && protocol.equals("https")) { clientCertNickname = nick; } } } catch (ParseException e) { System.err.println("Error in parsing command line options: " + e.getMessage()); usage(options); } CryptoManager manager = null; CryptoToken token = null; // Initialize token try { CryptoManager.initialize(db_dir); } catch (AlreadyInitializedException e) { // it is ok if it is already initialized } catch (Exception e) { log("INITIALIZATION ERROR: " + e.toString()); System.exit(1); } // log into token try { manager = CryptoManager.getInstance(); token = manager.getInternalKeyStorageToken(); Password password = new Password(token_pwd.toCharArray()); try { token.login(password); } catch (Exception e) { log("login Exception: " + e.toString()); if (!token.isLoggedIn()) { token.initPassword(password, password); } } } catch (Exception e) { log("Exception in logging into token:" + e.toString()); } CAClient client; CACertClient certClient; ProfileClient profileClient; try { ClientConfig config = new ClientConfig(); config.setServerURL(protocol + "://" + host + ":" + port); config.setCertNickname(clientCertNickname); client = new CAClient(new PKIClient(config, null)); certClient = (CACertClient) client.getClient("cert"); profileClient = (ProfileClient) client.getClient("profile"); } catch (Exception e) { e.printStackTrace(); return; } Collection<CertRequestInfo> list = null; try { list = certClient.listRequests("complete", null, null, null, null, null).getEntries(); } catch (Exception e) { e.printStackTrace(); } printRequests(list); //Get a CertInfo int certIdToPrint = 1; CertId id = new CertId(certIdToPrint); CertData certData = null; try { certData = certClient.getCert(id); } catch (CertNotFoundException e) { e.printStackTrace(); log("Cert: " + certIdToPrint + " not found. \n" + e.toString()); } printCertificate(certData); //Try an invalid Cert to print out //Get a CertInfo int certIdBadToPrint = 9999999; CertId certIdBad = new CertId(certIdBadToPrint); CertData certDataBad = null; try { certDataBad = certClient.getCert(certIdBad); } catch (CertNotFoundException e) { e.printStackTrace(); log("Cert: " + certIdBadToPrint + " not found. \n" + e.toString()); } printCertificate(certDataBad); //Get a CertInfoList CertDataInfos infos = null; try { infos = certClient.listCerts("VALID", null, null, null, null); } catch (Exception e) { e.printStackTrace(); } printCertInfos(infos, "no search filter:"); //Initiate a Certificate Enrollment CertEnrollmentRequest data = createUserCertEnrollment(); enrollAndApproveCertRequest(certClient, data); // submit a RA authenticated user cert request CertEnrollmentRequest rdata = createRAUserCertEnrollment(); enrollCertRequest(certClient, rdata); // now try a manually approved server cert CertEnrollmentRequest serverData = createServerCertEnrollment(); enrollAndApproveCertRequest(certClient, serverData); // submit using an agent approval profile serverData.setProfileId("caAgentServerCert"); enrollCertRequest(certClient, serverData); //Perform a sample certificate search with advanced search terms CertSearchRequest searchData = new CertSearchRequest(); searchData.setSerialNumberRangeInUse(true); searchData.setSerialFrom("9999"); searchData.setSerialTo("99990"); infos = certClient.findCerts(searchData, 100, 10); printCertInfos(infos, new FilterBuilder(searchData).buildFilter()); // Try to get a non existing request RequestId idBad = new RequestId("999999"); CertRequestInfo infoBad = null; try { infoBad = certClient.getRequest(idBad); } catch (RequestNotFoundException e) { e.printStackTrace(); log("Exception getting request #: " + idBad.toString() + "\n" + e.toString()); } printRequestInfo(infoBad); //Perform another sample certificate search with advanced search terms searchData = new CertSearchRequest(); searchData.setSubjectInUse(true); searchData.setEmail("jmagne@redhat.com"); searchData.setMatchExactly(true); infos = certClient.findCerts(searchData, 100, 10); printCertInfos(infos, new FilterBuilder(searchData).buildFilter()); //Get a list of Profiles ProfileDataInfos pInfos = profileClient.listProfiles(null, null); printProfileInfos(pInfos); // Get a specific profile String pId = "caUserCert"; ProfileData pData = profileClient.retrieveProfile(pId); printProfileData(pData); }
From source file:com.searchcode.app.App.java
public static void main(String[] args) { injector = Guice.createInjector(new InjectorConfig()); int server_port = Helpers.tryParseInt( Properties.getProperties().getProperty(Values.SERVERPORT, Values.DEFAULTSERVERPORT), Values.DEFAULTSERVERPORT);/* www . j a v a 2 s. c o m*/ boolean onlyLocalhost = Boolean .parseBoolean(Properties.getProperties().getProperty("only_localhost", "false")); // Database migrations happen before we start databaseMigrations(); LOGGER.info("Starting searchcode server on port " + server_port); Spark.port(server_port); JobService js = injector.getInstance(JobService.class); Repo repo = injector.getInstance(Repo.class); Data data = injector.getInstance(Data.class); Api api = injector.getInstance(Api.class); ApiService apiService = injector.getInstance(ApiService.class); StatsService statsService = new StatsService(); scl = Singleton.getSearchcodeLib(data); js.initialJobs(); Gson gson = new Gson(); Spark.staticFileLocation("/public"); before((request, response) -> { if (onlyLocalhost) { if (!request.ip().equals("127.0.0.1")) { halt(204); } } }); get("/", (req, res) -> { Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); if (req.queryParams().contains("q") && !req.queryParams("q").trim().equals("")) { String query = req.queryParams("q").trim(); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } List<String> reposList = new ArrayList<>(); List<String> langsList = new ArrayList<>(); List<String> ownsList = new ArrayList<>(); if (req.queryParams().contains("repo")) { String[] repos = new String[0]; repos = req.queryParamsValues("repo"); if (repos.length != 0) { reposList = Arrays.asList(repos); } } if (req.queryParams().contains("lan")) { String[] langs = new String[0]; langs = req.queryParamsValues("lan"); if (langs.length != 0) { langsList = Arrays.asList(langs); } } if (req.queryParams().contains("own")) { String[] owns = new String[0]; owns = req.queryParamsValues("own"); if (owns.length != 0) { ownsList = Arrays.asList(owns); } } map.put("searchValue", query); map.put("searchResultJson", gson.toJson(new CodePreload(query, page, langsList, reposList, ownsList))); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "search_test.ftl"); } // Totally pointless vanity but lets rotate the image every week int photoId = getWeekOfMonth(); if (photoId <= 0) { photoId = 3; } if (photoId > 4) { photoId = 2; } CodeSearcher cs = new CodeSearcher(); map.put("photoId", photoId); map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "index.ftl"); }, new FreeMarkerEngine()); get("/html/", (req, res) -> { CodeSearcher cs = new CodeSearcher(); CodeMatcher cm = new CodeMatcher(data); Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); if (req.queryParams().contains("q")) { String query = req.queryParams("q").trim(); String altquery = query.replaceAll("[^A-Za-z0-9 ]", " ").trim().replaceAll(" +", " "); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } String[] repos = new String[0]; String[] langs = new String[0]; String reposFilter = ""; String langsFilter = ""; String reposQueryString = ""; String langsQueryString = ""; if (req.queryParams().contains("repo")) { repos = req.queryParamsValues("repo"); if (repos.length != 0) { List<String> reposList = Arrays.asList(repos).stream() .map((s) -> "reponame:" + QueryParser.escape(s)).collect(Collectors.toList()); reposFilter = " && (" + StringUtils.join(reposList, " || ") + ")"; List<String> reposQueryList = Arrays.asList(repos).stream() .map((s) -> "&repo=" + URLEncoder.encode(s)).collect(Collectors.toList()); reposQueryString = StringUtils.join(reposQueryList, ""); } } if (req.queryParams().contains("lan")) { langs = req.queryParamsValues("lan"); if (langs.length != 0) { List<String> langsList = Arrays.asList(langs).stream() .map((s) -> "languagename:" + QueryParser.escape(s)).collect(Collectors.toList()); langsFilter = " && (" + StringUtils.join(langsList, " || ") + ")"; List<String> langsQueryList = Arrays.asList(langs).stream() .map((s) -> "&lan=" + URLEncoder.encode(s)).collect(Collectors.toList()); langsQueryString = StringUtils.join(langsQueryList, ""); } } // split the query escape it and and it together String cleanQueryString = scl.formatQueryString(query); SearchResult searchResult = cs.search(cleanQueryString + reposFilter + langsFilter, page); searchResult.setCodeResultList(cm.formatResults(searchResult.getCodeResultList(), query, true)); for (CodeFacetRepo f : searchResult.getRepoFacetResults()) { if (Arrays.asList(repos).contains(f.getRepoName())) { f.setSelected(true); } } for (CodeFacetLanguage f : searchResult.getLanguageFacetResults()) { if (Arrays.asList(langs).contains(f.getLanguageName())) { f.setSelected(true); } } map.put("searchValue", query); map.put("searchResult", searchResult); map.put("reposQueryString", reposQueryString); map.put("langsQueryString", langsQueryString); map.put("altQuery", altquery); map.put("isHtml", true); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "searchresults.ftl"); } // Totally pointless vanity but lets rotate the image every week int photoId = getWeekOfMonth(); if (photoId <= 0) { photoId = 3; } if (photoId > 4) { photoId = 2; } map.put("photoId", photoId); map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "index.ftl"); }, new FreeMarkerEngine()); /** * Allows one to write literal lucene search queries against the index * TODO This is still very much WIP */ get("/literal/", (req, res) -> { CodeSearcher cs = new CodeSearcher(); CodeMatcher cm = new CodeMatcher(data); Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); if (req.queryParams().contains("q")) { String query = req.queryParams("q").trim(); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } String altquery = query.replaceAll("[^A-Za-z0-9 ]", " ").trim().replaceAll(" +", " "); SearchResult searchResult = cs.search(query, page); searchResult.setCodeResultList(cm.formatResults(searchResult.getCodeResultList(), altquery, false)); map.put("searchValue", query); map.put("searchResult", searchResult); map.put("reposQueryString", ""); map.put("langsQueryString", ""); map.put("altQuery", ""); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "searchresults.ftl"); } map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "index.ftl"); }, new FreeMarkerEngine()); /** * This is the endpoint used by the frontend. */ get("/api/codesearch/", (req, res) -> { CodeSearcher cs = new CodeSearcher(); CodeMatcher cm = new CodeMatcher(data); if (req.queryParams().contains("q") && req.queryParams("q").trim() != "") { String query = req.queryParams("q").trim(); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } String[] repos = new String[0]; String[] langs = new String[0]; String[] owners = new String[0]; String reposFilter = ""; String langsFilter = ""; String ownersFilter = ""; if (req.queryParams().contains("repo")) { repos = req.queryParamsValues("repo"); if (repos.length != 0) { List<String> reposList = Arrays.asList(repos).stream() .map((s) -> "reponame:" + QueryParser.escape(s)).collect(Collectors.toList()); reposFilter = " && (" + StringUtils.join(reposList, " || ") + ")"; } } if (req.queryParams().contains("lan")) { langs = req.queryParamsValues("lan"); if (langs.length != 0) { List<String> langsList = Arrays.asList(langs).stream() .map((s) -> "languagename:" + QueryParser.escape(s)).collect(Collectors.toList()); langsFilter = " && (" + StringUtils.join(langsList, " || ") + ")"; } } if (req.queryParams().contains("own")) { owners = req.queryParamsValues("own"); if (owners.length != 0) { List<String> ownersList = Arrays.asList(owners).stream() .map((s) -> "codeowner:" + QueryParser.escape(s)).collect(Collectors.toList()); ownersFilter = " && (" + StringUtils.join(ownersList, " || ") + ")"; } } // Need to pass in the filters into this query String cacheKey = query + page + reposFilter + langsFilter + ownersFilter; if (cache.containsKey(cacheKey)) { return cache.get(cacheKey); } // split the query escape it and and it together String cleanQueryString = scl.formatQueryString(query); SearchResult searchResult = cs.search(cleanQueryString + reposFilter + langsFilter + ownersFilter, page); searchResult.setCodeResultList(cm.formatResults(searchResult.getCodeResultList(), query, true)); searchResult.setQuery(query); for (String altQuery : scl.generateAltQueries(query)) { searchResult.addAltQuery(altQuery); } // Null out code as it isnt required and there is no point in bloating our ajax requests for (CodeResult codeSearchResult : searchResult.getCodeResultList()) { codeSearchResult.setCode(null); } cache.put(cacheKey, searchResult); return searchResult; } return null; }, new JsonTransformer()); get("/api/repo/add/", "application/json", (request, response) -> { boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); if (!apiEnabled) { return new ApiResponse(false, "API not enabled"); } String publicKey = request.queryParams("pub"); String signedKey = request.queryParams("sig"); String reponames = request.queryParams("reponame"); String repourls = request.queryParams("repourl"); String repotype = request.queryParams("repotype"); String repousername = request.queryParams("repousername"); String repopassword = request.queryParams("repopassword"); String reposource = request.queryParams("reposource"); String repobranch = request.queryParams("repobranch"); if (reponames == null || reponames.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "reponame is a required parameter"); } if (repourls == null || repourls.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "repourl is a required parameter"); } if (repotype == null) { return new ApiResponse(false, "repotype is a required parameter"); } if (repousername == null) { return new ApiResponse(false, "repousername is a required parameter"); } if (repopassword == null) { return new ApiResponse(false, "repopassword is a required parameter"); } if (reposource == null) { return new ApiResponse(false, "reposource is a required parameter"); } if (repobranch == null) { return new ApiResponse(false, "repobranch is a required parameter"); } if (apiAuth) { if (publicKey == null || publicKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "pub is a required parameter"); } if (signedKey == null || signedKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "sig is a required parameter"); } String toValidate = String.format( "pub=%s&reponame=%s&repourl=%s&repotype=%s&repousername=%s&repopassword=%s&reposource=%s&repobranch=%s", URLEncoder.encode(publicKey), URLEncoder.encode(reponames), URLEncoder.encode(repourls), URLEncoder.encode(repotype), URLEncoder.encode(repousername), URLEncoder.encode(repopassword), URLEncoder.encode(reposource), URLEncoder.encode(repobranch)); boolean validRequest = apiService.validateRequest(publicKey, signedKey, toValidate); if (!validRequest) { return new ApiResponse(false, "invalid signed url"); } } // Clean if (repobranch == null || repobranch.trim().equals(Values.EMPTYSTRING)) { repobranch = "master"; } repotype = repotype.trim().toLowerCase(); if (!"git".equals(repotype) && !"svn".equals(repotype)) { repotype = "git"; } RepoResult repoResult = repo.getRepoByName(reponames); if (repoResult != null) { return new ApiResponse(false, "repository name already exists"); } repo.saveRepo(new RepoResult(-1, reponames, repotype, repourls, repousername, repopassword, reposource, repobranch)); return new ApiResponse(true, "added repository successfully"); }, new JsonTransformer()); get("/api/repo/delete/", "application/json", (request, response) -> { boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); if (!apiEnabled) { return new ApiResponse(false, "API not enabled"); } String publicKey = request.queryParams("pub"); String signedKey = request.queryParams("sig"); String reponames = request.queryParams("reponame"); if (reponames == null || reponames.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "reponame is a required parameter"); } if (apiAuth) { if (publicKey == null || publicKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "pub is a required parameter"); } if (signedKey == null || signedKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "sig is a required parameter"); } String toValidate = String.format("pub=%s&reponame=%s", URLEncoder.encode(publicKey), URLEncoder.encode(reponames)); boolean validRequest = apiService.validateRequest(publicKey, signedKey, toValidate); if (!validRequest) { return new ApiResponse(false, "invalid signed url"); } } RepoResult rr = repo.getRepoByName(reponames); if (rr == null) { return new ApiResponse(false, "repository already deleted"); } Singleton.getUniqueDeleteRepoQueue().add(rr); return new ApiResponse(true, "repository queued for deletion"); }, new JsonTransformer()); get("/api/repo/list/", "application/json", (request, response) -> { boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); if (!apiEnabled) { return new ApiResponse(false, "API not enabled"); } String publicKey = request.queryParams("pub"); String signedKey = request.queryParams("sig"); if (apiAuth) { if (publicKey == null || publicKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "pub is a required parameter"); } if (signedKey == null || signedKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "sig is a required parameter"); } String toValidate = String.format("pub=%s", URLEncoder.encode(publicKey)); boolean validRequest = apiService.validateRequest(publicKey, signedKey, toValidate); if (!validRequest) { return new ApiResponse(false, "invalid signed url"); } } List<RepoResult> repoResultList = repo.getAllRepo(); return new RepoResultApiResponse(true, Values.EMPTYSTRING, repoResultList); }, new JsonTransformer()); get("/admin/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } CodeSearcher cs = new CodeSearcher(); Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("numSearches", statsService.getSearchCount()); map.put("uptime", statsService.getUptime()); // Put all properties here map.put(Values.SQLITEFILE, Properties.getProperties().getProperty(Values.SQLITEFILE, Values.DEFAULTSQLITEFILE)); map.put(Values.SERVERPORT, Properties.getProperties().getProperty(Values.SERVERPORT, Values.DEFAULTSERVERPORT)); map.put(Values.REPOSITORYLOCATION, Properties.getProperties().getProperty(Values.REPOSITORYLOCATION, Values.DEFAULTREPOSITORYLOCATION)); map.put(Values.INDEXLOCATION, Properties.getProperties().getProperty(Values.INDEXLOCATION, Values.DEFAULTINDEXLOCATION)); map.put(Values.FACETSLOCATION, Properties.getProperties().getProperty(Values.FACETSLOCATION, Values.DEFAULTFACETSLOCATION)); map.put(Values.CHECKREPOCHANGES, Properties.getProperties().getProperty(Values.CHECKREPOCHANGES, Values.DEFAULTCHECKREPOCHANGES)); map.put(Values.ONLYLOCALHOST, Properties.getProperties().getProperty(Values.ONLYLOCALHOST, Values.DEFAULTONLYLOCALHOST)); map.put(Values.LOWMEMORY, Properties.getProperties().getProperty(Values.LOWMEMORY, Values.DEFAULTLOWMEMORY)); map.put(Values.SPELLINGCORRECTORSIZE, Properties.getProperties() .getProperty(Values.SPELLINGCORRECTORSIZE, Values.DEFAULTSPELLINGCORRECTORSIZE)); map.put(Values.USESYSTEMGIT, Properties.getProperties().getProperty(Values.USESYSTEMGIT, Values.DEFAULTUSESYSTEMGIT)); map.put(Values.GITBINARYPATH, Properties.getProperties().getProperty(Values.GITBINARYPATH, Values.DEFAULTGITBINARYPATH)); map.put(Values.APIENABLED, Properties.getProperties().getProperty(Values.APIENABLED, Values.DEFAULTAPIENABLED)); map.put(Values.APIKEYAUTH, Properties.getProperties().getProperty(Values.APIKEYAUTH, Values.DEFAULTAPIKEYAUTH)); map.put(Values.SVNBINARYPATH, Properties.getProperties().getProperty(Values.SVNBINARYPATH, Values.DEFAULTSVNBINARYPATH)); map.put(Values.SVNENABLED, Properties.getProperties().getProperty(Values.SVNENABLED, Values.DEFAULTSVNENABLED)); map.put(Values.MAXDOCUMENTQUEUESIZE, Properties.getProperties().getProperty(Values.MAXDOCUMENTQUEUESIZE, Values.DEFAULTMAXDOCUMENTQUEUESIZE)); map.put(Values.MAXDOCUMENTQUEUELINESIZE, Properties.getProperties() .getProperty(Values.MAXDOCUMENTQUEUELINESIZE, Values.DEFAULTMAXDOCUMENTQUEUELINESIZE)); map.put(Values.MAXFILELINEDEPTH, Properties.getProperties().getProperty(Values.MAXFILELINEDEPTH, Values.DEFAULTMAXFILELINEDEPTH)); map.put("deletionQueue", Singleton.getUniqueDeleteRepoQueue().size()); map.put("version", VERSION); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin.ftl"); }, new FreeMarkerEngine()); get("/admin/repo/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } int repoCount = repo.getRepoCount(); String offSet = request.queryParams("offset"); String searchQuery = request.queryParams("q"); int indexOffset = 0; Map<String, Object> map = new HashMap<>(); if (offSet != null) { try { indexOffset = Integer.parseInt(offSet); if (indexOffset > repoCount || indexOffset < 0) { indexOffset = 0; } } catch (NumberFormatException ex) { indexOffset = 0; } } if (searchQuery != null) { map.put("repoResults", repo.searchRepo(searchQuery)); } else { map.put("repoResults", repo.getPagedRepo(indexOffset, 100)); } map.put("searchQuery", searchQuery); map.put("hasPrevious", indexOffset > 0); map.put("hasNext", (indexOffset + 100) < repoCount); map.put("previousOffset", "" + (indexOffset - 100)); map.put("nextOffset", "" + (indexOffset + 100)); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_repo.ftl"); }, new FreeMarkerEngine()); get("/admin/bulk/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_bulk.ftl"); }, new FreeMarkerEngine()); get("/admin/api/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("apiKeys", api.getAllApi()); boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); map.put("apiAuthentication", apiEnabled && apiAuth); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_api.ftl"); }, new FreeMarkerEngine()); post("/admin/api/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } apiService.createKeys(); response.redirect("/admin/api/"); halt(); return null; }, new FreeMarkerEngine()); get("/admin/api/delete/", "application/json", (request, response) -> { if (getAuthenticatedUser(request) == null || !request.queryParams().contains("publicKey")) { response.redirect("/login/"); halt(); return false; } String publicKey = request.queryParams("publicKey"); apiService.deleteKey(publicKey); return true; }, new JsonTransformer()); get("/admin/settings/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } String[] highlighters = "agate,androidstudio,arta,ascetic,atelier-cave.dark,atelier-cave.light,atelier-dune.dark,atelier-dune.light,atelier-estuary.dark,atelier-estuary.light,atelier-forest.dark,atelier-forest.light,atelier-heath.dark,atelier-heath.light,atelier-lakeside.dark,atelier-lakeside.light,atelier-plateau.dark,atelier-plateau.light,atelier-savanna.dark,atelier-savanna.light,atelier-seaside.dark,atelier-seaside.light,atelier-sulphurpool.dark,atelier-sulphurpool.light,brown_paper,codepen-embed,color-brewer,dark,darkula,default,docco,far,foundation,github-gist,github,googlecode,grayscale,hopscotch,hybrid,idea,ir_black,kimbie.dark,kimbie.light,magula,mono-blue,monokai,monokai_sublime,obsidian,paraiso.dark,paraiso.light,pojoaque,railscasts,rainbow,school_book,solarized_dark,solarized_light,sunburst,tomorrow-night-blue,tomorrow-night-bright,tomorrow-night-eighties,tomorrow-night,tomorrow,vs,xcode,zenburn" .split(","); Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("syntaxHighlighter", getSyntaxHighlighter()); map.put("highlighters", highlighters); map.put("averageSalary", "" + (int) getAverageSalary()); map.put("matchLines", "" + (int) getMatchLines()); map.put("maxLineDepth", "" + (int) getMaxLineDepth()); map.put("minifiedLength", "" + (int) getMinifiedLength()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_settings.ftl"); }, new FreeMarkerEngine()); get("/admin/reports/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_reports.ftl"); }, new FreeMarkerEngine()); post("/admin/settings/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } if (ISCOMMUNITY) { response.redirect("/admin/settings/"); halt(); } String logo = request.queryParams("logo").trim(); String syntaxHighlighter = request.queryParams("syntaxhighligher"); try { double averageSalary = Double.parseDouble(request.queryParams("averagesalary")); data.saveData(Values.AVERAGESALARY, "" + (int) averageSalary); } catch (NumberFormatException ex) { data.saveData(Values.AVERAGESALARY, Values.DEFAULTAVERAGESALARY); } try { double averageSalary = Double.parseDouble(request.queryParams("matchlines")); data.saveData(Values.MATCHLINES, "" + (int) averageSalary); } catch (NumberFormatException ex) { data.saveData(Values.MATCHLINES, Values.DEFAULTMATCHLINES); } try { double averageSalary = Double.parseDouble(request.queryParams("maxlinedepth")); data.saveData(Values.MAXLINEDEPTH, "" + (int) averageSalary); } catch (NumberFormatException ex) { data.saveData(Values.MAXLINEDEPTH, Values.DEFAULTMAXLINEDEPTH); } try { double minifiedlength = Double.parseDouble(request.queryParams("minifiedlength")); data.saveData(Values.MINIFIEDLENGTH, "" + (int) minifiedlength); } catch (NumberFormatException ex) { data.saveData(Values.MINIFIEDLENGTH, Values.DEFAULTMINIFIEDLENGTH); } data.saveData(Values.LOGO, logo); data.saveData(Values.SYNTAXHIGHLIGHTER, syntaxHighlighter); // Redo anything that requires updates at this point scl = Singleton.getSearchcodeLib(data); response.redirect("/admin/settings/"); halt(); return null; }, new FreeMarkerEngine()); post("/admin/bulk/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } String repos = request.queryParams("repos"); String repolines[] = repos.split("\\r?\\n"); for (String line : repolines) { String[] repoparams = line.split(",", -1); if (repoparams.length == 7) { String branch = repoparams[6].trim(); if (branch.equals(Values.EMPTYSTRING)) { branch = "master"; } String scm = repoparams[1].trim().toLowerCase(); if (scm.equals(Values.EMPTYSTRING)) { scm = "git"; } RepoResult rr = repo.getRepoByName(repoparams[0]); if (rr == null) { repo.saveRepo(new RepoResult(-1, repoparams[0], scm, repoparams[2], repoparams[3], repoparams[4], repoparams[5], branch)); } } } response.redirect("/admin/bulk/"); halt(); return null; }, new FreeMarkerEngine()); post("/admin/repo/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } String[] reponames = request.queryParamsValues("reponame"); String[] reposcms = request.queryParamsValues("reposcm"); String[] repourls = request.queryParamsValues("repourl"); String[] repousername = request.queryParamsValues("repousername"); String[] repopassword = request.queryParamsValues("repopassword"); String[] reposource = request.queryParamsValues("reposource"); String[] repobranch = request.queryParamsValues("repobranch"); for (int i = 0; i < reponames.length; i++) { if (reponames[i].trim().length() != 0) { String branch = repobranch[i].trim(); if (branch.equals(Values.EMPTYSTRING)) { branch = "master"; } repo.saveRepo(new RepoResult(-1, reponames[i], reposcms[i], repourls[i], repousername[i], repopassword[i], reposource[i], branch)); } } response.redirect("/admin/repo/"); halt(); return null; }, new FreeMarkerEngine()); get("/login/", (request, response) -> { if (getAuthenticatedUser(request) != null) { response.redirect("/admin/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "login.ftl"); }, new FreeMarkerEngine()); post("/login/", (req, res) -> { if (req.queryParams().contains("password") && req.queryParams("password") .equals(com.searchcode.app.util.Properties.getProperties().getProperty("password"))) { addAuthenticatedUser(req); res.redirect("/admin/"); halt(); } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "login.ftl"); }, new FreeMarkerEngine()); get("/logout/", (req, res) -> { removeAuthenticatedUser(req); res.redirect("/"); return null; }); get("/admin/delete/", "application/json", (request, response) -> { if (getAuthenticatedUser(request) == null || !request.queryParams().contains("repoName")) { response.redirect("/login/"); halt(); return false; } String repoName = request.queryParams("repoName"); RepoResult rr = repo.getRepoByName(repoName); if (rr != null) { Singleton.getUniqueDeleteRepoQueue().add(rr); } return true; }, new JsonTransformer()); get("/admin/checkversion/", "application/json", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return false; } String version; try { version = IOUtils.toString(new URL("https://searchcode.com/product/version/")).replace("\"", Values.EMPTYSTRING); } catch (IOException ex) { return "Unable to determine if running the latest version. Check https://searchcode.com/product/download/ for the latest release."; } if (App.VERSION.equals(version)) { return "Your searchcode server version " + version + " is the latest."; } else { return "Your searchcode server version " + App.VERSION + " instance is out of date. The latest version is " + version + "."; } }, new JsonTransformer()); get("/file/:codeid/:reponame/*", (request, response) -> { Map<String, Object> map = new HashMap<>(); CodeSearcher cs = new CodeSearcher(); Cocomo2 coco = new Cocomo2(); StringBuilder code = new StringBuilder(); String fileName = Values.EMPTYSTRING; if (request.splat().length != 0) { fileName = request.splat()[0]; } CodeResult codeResult = cs.getByRepoFileName(request.params(":reponame"), fileName); if (codeResult == null) { int codeid = Integer.parseInt(request.params(":codeid")); codeResult = cs.getById(codeid); } if (codeResult == null) { response.redirect("/404/"); halt(); } List<String> codeLines = codeResult.code; for (int i = 0; i < codeLines.size(); i++) { code.append("<span id=\"" + (i + 1) + "\"></span>"); code.append(StringEscapeUtils.escapeHtml4(codeLines.get(i))); code.append("\n"); } boolean highlight = true; if (Integer.parseInt(codeResult.codeLines) > 1000) { highlight = false; } RepoResult repoResult = repo.getRepoByName(codeResult.repoName); if (repoResult != null) { map.put("source", repoResult.getSource()); } map.put("fileName", codeResult.fileName); map.put("codePath", codeResult.codePath); map.put("codeLength", codeResult.codeLines); map.put("languageName", codeResult.languageName); map.put("md5Hash", codeResult.md5hash); map.put("repoName", codeResult.repoName); map.put("highlight", highlight); map.put("repoLocation", codeResult.getRepoLocation()); map.put("codeValue", code.toString()); map.put("highligher", getSyntaxHighlighter()); map.put("codeOwner", codeResult.getCodeOwner()); double estimatedEffort = coco.estimateEffort(scl.countFilteredLines(codeResult.getCode())); int estimatedCost = (int) coco.estimateCost(estimatedEffort, getAverageSalary()); if (estimatedCost != 0 && !scl.languageCostIgnore(codeResult.getLanguageName())) { map.put("estimatedCost", estimatedCost); } map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "coderesult.ftl"); }, new FreeMarkerEngine()); /** * Deprecated should not be used * TODO delete this method */ get("/codesearch/view/:codeid", (request, response) -> { Map<String, Object> map = new HashMap<>(); int codeid = Integer.parseInt(request.params(":codeid")); CodeSearcher cs = new CodeSearcher(); Cocomo2 coco = new Cocomo2(); StringBuilder code = new StringBuilder(); // escape all the lines and include deeplink for line number CodeResult codeResult = cs.getById(codeid); if (codeResult == null) { response.redirect("/404/"); halt(); } List<String> codeLines = codeResult.code; for (int i = 0; i < codeLines.size(); i++) { code.append("<span id=\"" + (i + 1) + "\"></span>"); code.append(StringEscapeUtils.escapeHtml4(codeLines.get(i))); code.append("\n"); } boolean highlight = true; if (Integer.parseInt(codeResult.codeLines) > 1000) { highlight = false; } RepoResult repoResult = repo.getRepoByName(codeResult.repoName); if (repoResult != null) { map.put("source", repoResult.getSource()); } map.put("fileName", codeResult.fileName); map.put("codePath", codeResult.codePath); map.put("codeLength", codeResult.codeLines); map.put("languageName", codeResult.languageName); map.put("md5Hash", codeResult.md5hash); map.put("repoName", codeResult.repoName); map.put("highlight", highlight); map.put("repoLocation", codeResult.getRepoLocation()); map.put("codeValue", code.toString()); map.put("highligher", getSyntaxHighlighter()); map.put("codeOwner", codeResult.getCodeOwner()); double estimatedEffort = coco.estimateEffort(scl.countFilteredLines(codeResult.getCode())); int estimatedCost = (int) coco.estimateCost(estimatedEffort, getAverageSalary()); if (estimatedCost != 0 && !scl.languageCostIgnore(codeResult.getLanguageName())) { map.put("estimatedCost", estimatedCost); } map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "coderesult.ftl"); }, new FreeMarkerEngine()); get("/documentation/", (request, response) -> { Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "documentation.ftl"); }, new FreeMarkerEngine()); get("/search_test/", (request, response) -> { Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "search_test.ftl"); }, new FreeMarkerEngine()); get("/404/", (request, response) -> { Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "404.ftl"); }, new FreeMarkerEngine()); /** * Test that was being used to display blame information */ // get("/test/:reponame/*", (request, response) -> { // User user = injector.getInstance(User.class); // user.Blame(request.params(":reponame"), request.splat()[0]); // return ""; // }, new JsonTransformer()); }
From source file:edu.msu.cme.rdp.classifier.ClassifierCmd.java
/** * This is the main method to do classification. * <p>Usage: java ClassifierCmd queryFile outputFile [property file]. * <br>//w ww.j av a 2 s . c om * queryFile can be one of the following formats: Fasta, Genbank and EMBL. * <br> * outputFile will be used to save the classification output. * <br> * property file contains the mapping of the training files. * <br> * Note: the training files and the property file should be in the same directory. * The default property file is set to data/classifier/16srrna/rRNAClassifier.properties. */ public static void main(String[] args) throws Exception { String queryFile = null; String outputFile = null; String propFile = null; String gene = null; ClassificationResultFormatter.FORMAT format = CmdOptions.DEFAULT_FORMAT; int min_bootstrap_words = Classifier.MIN_BOOTSTRSP_WORDS; try { CommandLine line = new PosixParser().parse(options, args); if (line.hasOption(CmdOptions.OUTFILE_SHORT_OPT)) { outputFile = line.getOptionValue(CmdOptions.OUTFILE_SHORT_OPT); } else { throw new Exception("outputFile must be specified"); } if (line.hasOption(CmdOptions.TRAINPROPFILE_SHORT_OPT)) { if (gene != null) { throw new IllegalArgumentException( "Already specified the gene from the default location. Can not specify train_propfile"); } else { propFile = line.getOptionValue(CmdOptions.TRAINPROPFILE_SHORT_OPT); } } if (line.hasOption(CmdOptions.FORMAT_SHORT_OPT)) { String f = line.getOptionValue(CmdOptions.FORMAT_SHORT_OPT); if (f.equalsIgnoreCase("allrank")) { format = ClassificationResultFormatter.FORMAT.allRank; } else if (f.equalsIgnoreCase("fixrank")) { format = ClassificationResultFormatter.FORMAT.fixRank; } else if (f.equalsIgnoreCase("filterbyconf")) { format = ClassificationResultFormatter.FORMAT.filterbyconf; } else if (f.equalsIgnoreCase("db")) { format = ClassificationResultFormatter.FORMAT.dbformat; } else { throw new IllegalArgumentException( "Not valid output format, only allrank, fixrank, filterbyconf and db allowed"); } } if (line.hasOption(CmdOptions.GENE_SHORT_OPT)) { if (propFile != null) { throw new IllegalArgumentException( "Already specified train_propfile. Can not specify gene any more"); } gene = line.getOptionValue(CmdOptions.GENE_SHORT_OPT).toLowerCase(); if (!gene.equals(ClassifierFactory.RRNA_16S_GENE) && !gene.equals(ClassifierFactory.FUNGALLSU_GENE)) { throw new IllegalArgumentException(gene + " is NOT valid, only allows " + ClassifierFactory.RRNA_16S_GENE + " and " + ClassifierFactory.FUNGALLSU_GENE); } } if (line.hasOption(CmdOptions.MIN_BOOTSTRAP_WORDS_SHORT_OPT)) { min_bootstrap_words = Integer .parseInt(line.getOptionValue(CmdOptions.MIN_BOOTSTRAP_WORDS_SHORT_OPT)); if (min_bootstrap_words < Classifier.MIN_BOOTSTRSP_WORDS) { throw new IllegalArgumentException(CmdOptions.MIN_BOOTSTRAP_WORDS_LONG_OPT + " must be at least " + Classifier.MIN_BOOTSTRSP_WORDS); } } args = line.getArgs(); if (args.length != 1) { throw new Exception("Expect one query file"); } queryFile = args[0]; } catch (Exception e) { System.out.println("Command Error: " + e.getMessage()); new HelpFormatter().printHelp(120, "ClassifierCmd [options] <samplefile>\nNote this is the legacy command for one sample classification ", "", options, ""); return; } if (propFile == null && gene == null) { gene = CmdOptions.DEFAULT_GENE; } ClassifierCmd classifierCmd = new ClassifierCmd(); printLicense(); classifierCmd.doClassify(queryFile, outputFile, propFile, format, gene, min_bootstrap_words); }
From source file:com.ibm.crail.tools.CrailFsck.java
public static void main(String[] args) throws Exception { String type = ""; String filename = "/tmp.dat"; long offset = 0; long length = 1; boolean randomize = false; int storageClass = 0; int locationClass = 0; Option typeOption = Option.builder("t").desc( "type of experiment [getLocations|directoryDump|namenodeDump|blockStatistics|ping|createDirectory]") .hasArg().build();/*from w w w .j a va2 s . c o m*/ Option fileOption = Option.builder("f").desc("filename").hasArg().build(); Option offsetOption = Option.builder("y").desc("offset into the file").hasArg().build(); Option lengthOption = Option.builder("l").desc("length of the file [bytes]").hasArg().build(); Option storageOption = Option.builder("c").desc("storageClass for file [1..n]").hasArg().build(); Option locationOption = Option.builder("p").desc("locationClass for file [1..n]").hasArg().build(); Options options = new Options(); options.addOption(typeOption); options.addOption(fileOption); options.addOption(offsetOption); options.addOption(lengthOption); options.addOption(storageOption); options.addOption(locationOption); CommandLineParser parser = new DefaultParser(); CommandLine line = parser.parse(options, Arrays.copyOfRange(args, 0, args.length)); if (line.hasOption(typeOption.getOpt())) { type = line.getOptionValue(typeOption.getOpt()); } if (line.hasOption(fileOption.getOpt())) { filename = line.getOptionValue(fileOption.getOpt()); } if (line.hasOption(offsetOption.getOpt())) { offset = Long.parseLong(line.getOptionValue(offsetOption.getOpt())); } if (line.hasOption(lengthOption.getOpt())) { length = Long.parseLong(line.getOptionValue(lengthOption.getOpt())); } if (line.hasOption(storageOption.getOpt())) { storageClass = Integer.parseInt(line.getOptionValue(storageOption.getOpt())); } if (line.hasOption(locationOption.getOpt())) { locationClass = Integer.parseInt(line.getOptionValue(locationOption.getOpt())); } CrailFsck fsck = new CrailFsck(); if (type.equals("getLocations")) { fsck.getLocations(filename, offset, length); } else if (type.equals("directoryDump")) { fsck.directoryDump(filename, randomize); } else if (type.equals("namenodeDump")) { fsck.namenodeDump(); } else if (type.equals("blockStatistics")) { fsck.blockStatistics(filename); } else if (type.equals("ping")) { fsck.ping(); } else if (type.equals("createDirectory")) { fsck.createDirectory(filename, storageClass, locationClass); } else { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("crail fsck", options); System.exit(-1); } }
From source file:ca.uqac.dim.mapreduce.ltl.ParaLTLValidation.java
/** * Program entry point./*from w w w . j a v a 2 s. c om*/ * @param args Command-line arguments */ @SuppressWarnings("static-access") public static void main(String[] args) { // Define and process command line arguments Options options = new Options(); HelpFormatter help_formatter = new HelpFormatter(); Option opt; options.addOption("h", "help", false, "Show help"); opt = OptionBuilder.withArgName("property").hasArg() .withDescription("Property to verify, enclosed in double quotes").create("p"); options.addOption(opt); opt = OptionBuilder.withArgName("filename").hasArg().withDescription("Input filename").create("i"); options.addOption(opt); opt = OptionBuilder.withArgName("x").hasArg() .withDescription("Set verbosity level to x (default: 0 = quiet)").create("v"); options.addOption(opt); opt = OptionBuilder.withArgName("ParserType").hasArg().withDescription("Parser type (Dom or Sax)") .create("t"); options.addOption(opt); opt = OptionBuilder.withLongOpt("redirection").withArgName("x").hasArg() .withDescription("Set the redirection file for the System.out").create("r"); options.addOption(opt); opt = OptionBuilder.withLongOpt("mapper").withArgName("x").hasArg() .withDescription("Set the number of mapper").create("m"); options.addOption(opt); opt = OptionBuilder.withLongOpt("reducer").withArgName("x").hasArg() .withDescription("Set the number of reducer").create("n"); options.addOption(opt); CommandLine c_line = parseCommandLine(options, args); String redirectionFile = ""; //Contains a redirection file for the output if (c_line.hasOption("redirection")) { try { redirectionFile = c_line.getOptionValue("redirection"); PrintStream ps; ps = new PrintStream(redirectionFile); System.setOut(ps); } catch (FileNotFoundException e) { System.out.println("Redirection error !!!"); e.printStackTrace(); } } if (!c_line.hasOption("p") || !c_line.hasOption("i") | c_line.hasOption("h")) { help_formatter.printHelp(app_name, options); System.exit(1); } assert c_line.hasOption("p"); assert c_line.hasOption("i"); String trace_filename = c_line.getOptionValue("i"); String trace_format = getExtension(trace_filename); String property_str = c_line.getOptionValue("p"); String ParserType = ""; int MapperNum = 0; int ReducerNum = 0; //Contains a parser type if (c_line.hasOption("t")) { ParserType = c_line.getOptionValue("t"); } else { System.err.println("No Parser Type in Arguments"); System.exit(ERR_ARGUMENTS); } //Contains a mapper number if (c_line.hasOption("m")) { MapperNum = Integer.parseInt(c_line.getOptionValue("m")); } else { System.err.println("No Mapper Number in Arguments"); System.exit(ERR_ARGUMENTS); } //Contains a reducer number if (c_line.hasOption("n")) { ReducerNum = Integer.parseInt(c_line.getOptionValue("n")); } else { System.err.println("No Reducer Number in Arguments"); System.exit(ERR_ARGUMENTS); } if (c_line.hasOption("v")) m_verbosity = Integer.parseInt(c_line.getOptionValue("v")); // Obtain the property to verify and break into subformulas Operator property = null; try { int preset = Integer.parseInt(property_str); property = new Edoc2012Presets().property(preset); } catch (NumberFormatException e) { try { property = Operator.parseFromString(property_str); } catch (Operator.ParseException pe) { System.err.println("ERROR: parsing"); System.exit(1); } } Set<Operator> subformulas = property.getSubformulas(); // Initialize first collector depending on input file format int max_loops = property.getDepth(); int max_tuples_total = 0, total_tuples_total = 0; long time_begin = System.nanoTime(); TraceCollector initial_collector = null; { File in_file = new File(trace_filename); if (trace_format.compareToIgnoreCase(".txt") == 0) { initial_collector = new CharacterTraceCollector(in_file, subformulas); } else if (trace_format.compareToIgnoreCase(".xml") == 0) { if (ParserType.equals("Dom")) { initial_collector = new XmlDomTraceCollector(in_file, subformulas); } else if (ParserType.equals("Sax")) { initial_collector = new XmlSaxTraceCollector(in_file, subformulas); } else { initial_collector = new XmlSaxTraceCollector(in_file, subformulas); } } } if (initial_collector == null) { System.err.println("ERROR: unrecognized input format"); System.exit(1); } // Start workflow int trace_len = initial_collector.getTraceLength(); InCollector<Operator, LTLTupleValue> loop_collector = initial_collector; print(System.out, property.toString(), 2); print(System.out, loop_collector.toString(), 3); for (int i = 0; i < max_loops; i++) { print(System.out, "Loop " + i, 2); LTLParallelWorkflow w = new LTLParallelWorkflow(new LTLMapper(subformulas), new LTLReducer(subformulas, trace_len), loop_collector, new ResourceManager<Operator, LTLTupleValue>(MapperNum), new ResourceManager<Operator, LTLTupleValue>(ReducerNum)); loop_collector = w.run(); max_tuples_total += w.getMaxTuples(); total_tuples_total += w.getTotalTuples(); if (m_verbosity >= 3) { print(System.out, loop_collector.toString(), 3); } } boolean result = getVerdict(loop_collector, property); long time_end = System.nanoTime(); if (result) print(System.out, "Formula is true", 1); else print(System.out, "Formula is false", 1); long time_total = (time_end - time_begin) / 1000000; System.out.println(trace_len + "," + max_tuples_total + "," + total_tuples_total + "," + time_total); }
From source file:com.ikanow.infinit.e.application.utils.LogstashConfigUtils.java
public static void main(String[] args) throws IOException { System.out.println(Arrays.toString(args)); Globals.setIdentity(com.ikanow.infinit.e.data_model.Globals.Identity.IDENTITY_API); Globals.overrideConfigLocation(args[0]); // 1) Errored sources - things that break the formatting StringBuffer errors = new StringBuffer(); String testName;/*from w ww .j a va2 s. c o m*/ // 1.1) {} mismatch 1 //a errors.setLength(0); testName = "error_1_1a"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().startsWith("{} Mismatch (})")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } //b errors.setLength(0); testName = "error_1_1b"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().startsWith("{} Mismatch (})")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } //c errors.setLength(0); testName = "error_1_1c"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().startsWith("{} Mismatch (})")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // 1.2) {} mismatch 2 //a errors.setLength(0); testName = "error_1_2a"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().startsWith("{} Mismatch ({)")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // 1.3) multiple input/filter blocks // 1.3a) input errors.setLength(0); testName = "error_1_3a"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().equals("Multiple input or filter blocks: input")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // 1.3b) filter errors.setLength(0); testName = "error_1_3b"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().equals("Multiple input or filter blocks: filter")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // 1.4) unrecognized blocks // a output - special case errors.setLength(0); testName = "error_1_4a"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString() .equals("Not allowed output blocks - these are appended automatically by the logstash harvester")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // b errors.setLength(0); testName = "error_1_4b"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().equals("Unrecognized processing block: something_random")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // 1.5) fields/sub-elements that are not permitted // a ... sincedb_path errors.setLength(0); testName = "error_1_5a"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().equals("Not allowed sincedb_path in input.* block")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // b ... filter as sub-path of input errors.setLength(0); testName = "error_1_5b"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().equals("Not allowed sub-elements of input called 'filter' (1)")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // c ... filter as sub-path of sub-element of input errors.setLength(0); testName = "error_1_5c"; if (null != parseLogstashConfig(getTestFile(testName), errors)) { System.out.println("**** FAIL " + testName); } else if (!errors.toString().equals("Not allowed sub-elements of input called 'filter' (2)")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // 2) Valid formatted source BasicDBObject retVal; String output; String inputName; // (for re-using config files across text) //2.1) errors.setLength(0); testName = "success_2_1"; if (null == (retVal = parseLogstashConfig(getTestFile(testName), errors))) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } else if (!retVal.toString().equals( "{ \"input\" : { \"file\" : [ { \"path\" : { } , \"start_position\" : { } , \"type\" : { } , \"codec.multiline\" : { }}]} , \"filter\" : { \"csv\" : [ { \"columns\" : { }}] , \"drop\" : [ { }] , \"mutate\" : [ { \"convert\" : { }} , { \"add_fields\" : { }} , { \"rename\" : { }}] , \"date\" : [ { \"timezone\" : { } , \"match\" : { }}] , \"geoip\" : [ { \"source\" : { } , \"fields\" : { }}]}}")) { System.out.println("**** FAIL " + testName + ": " + retVal.toString()); } //System.out.println("(val="+retVal+")"); // 2.2 errors.setLength(0); testName = "success_2_2"; if (null == (retVal = parseLogstashConfig(getTestFile(testName), errors))) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } if (null == MongoDbUtil.getProperty(retVal, "filter.geoip.fields")) { System.out.println("**** FAIL " + testName + ": " + retVal); } //System.out.println(retVal); //2.3) - check that the sincedb is added correctly, plus the sourceKey manipulation // (USE success_2_1 for this) errors.setLength(0); testName = "inputs_2_3"; inputName = "success_2_3"; if (null == (output = validateLogstashInput(testName, getTestFile(inputName), errors, true))) { System.out.println("**** FAIL " + testName + ": errored: " + errors); } else { String outputToTest = output.replaceAll("[\n\r]", "\\\\n").replaceAll("\\s+", " "); String testAgainst = "input {\n\n file {\n sincedb_path => \"_XXX_DOTSINCEDB_XXX_\"\n\n\n path => \"/root/odin-poc-data/proxy_logs/may_known_cnc.csv\"\n\n start_position => beginning\n\n type => \"proxy_logs\"\n\n codec => multiline {\n\n pattern => \"^%{YEAR}-%{MONTHNUM}-%{MONTHDAY}%{DATA:summary}\"\n\n negate => true\n\n what => \"previous\"\n\n } \n\n add_field => [ \"sourceKey\", \"inputs_2_3\"] \n\n}\n\n}\n\n\n\nfilter { \n if [sourceKey] == \"inputs_2_3\" { \n\n \n\n if [type] == \"proxy_logs\" {\n\n csv {\n\n columns => [\"Device_Name\",\"SimpleDate\",\"Event_#Date\",\"Source_IP\",\"Source_Port\",\"Destination_IP\",\"Destination_Port\",\"Protocol\",\"Vendor_Alert\",\"MSS_Action\",\"Logging_Device_IP\",\"Application\",\"Bytes_Received\",\"Bytes_Sent\",\"Dest._Country\",\"Message\",\"Message_Type\",\"MSS_Log_Source_IP\",\"MSS_Log_Source_Type\",\"MSS_Log_Source_UUID\",\"network_protocol_id\",\"OS_Type\",\"PIX_Main-Code\",\"PIX_Sub-Code\",\"Port\",\"Product_ID\",\"Product\",\"Rule\",\"Rule_Identifier\",\"Sensor_Name\",\"Class\",\"Translate_Destination_IP\",\"Translate_Destination_Port\",\"Translate_Source_IP\"]\n\n }\n\n if [Device_Name] == \"Device Name\" {\n\n drop {}\n\n }\n\n mutate {\n\n convert => [ \"Bytes_Received\", \"integer\" ]\n\n convert => [ \"Bytes_Sent\", \"integer\" ]\n\n }\n\n date {\n\n timezone => \"Europe/London\"\n\n match => [ \"Event_Date\" , \"yyyy-MM-dd'T'HH:mm:ss\" ]\n\n }\n\n geoip {\n\n source => \"Destination_IP\"\n\n fields => [\"timezone\",\"location\",\"latitude\",\"longitude\"]\n\n }\n\n }\n\n\n\n mutate { update => [ \"sourceKey\", \"inputs_2_3\"] } \n}\n}\n"; testAgainst = testAgainst.replaceAll("[\n\r]", "\\\\n").replaceAll("\\s+", " "); if (!outputToTest.equals(testAgainst)) { System.out.println("**** FAIL " + testName + ": " + output); } } // 3) Valid formatted source, access to restricted types // 3.1) input // a) restricted - admin // (USE success_2_1 for this) errors.setLength(0); testName = "inputs_3_1a"; inputName = "success_2_1"; if (null != (output = validateLogstashInput(testName, getTestFile(inputName), errors, false))) { System.out.println("**** FAIL " + testName + ": Should have errored: " + output); } else if (!errors.toString() .startsWith("Security error, non-admin not allowed input type file, allowed options: ")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // b) restricted - non admin // (USE success_2_1 for this) errors.setLength(0); testName = "inputs_3_1b"; inputName = "success_2_1"; if (null == (output = validateLogstashInput(testName, getTestFile(inputName), errors, true))) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // c) unrestricted - non admin errors.setLength(0); testName = "inputs_3_1c"; inputName = "inputs_3_1c"; if (null == (output = validateLogstashInput(testName, getTestFile(inputName), errors, true))) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } //System.out.println("(val="+output+")"); // d) no input at all errors.setLength(0); testName = "inputs_3_1d"; inputName = "inputs_3_1d"; if (null != (output = validateLogstashInput(testName, getTestFile(inputName), errors, false))) { System.out.println("**** FAIL " + testName + ": Should have errored: " + output); } else if (!errors.toString().startsWith( "Invalid input format, should be 'input { INPUT_TYPE { ... } }' (only one INPUT_TYPE) and also contain a filter, no \"s around them.")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } // 3.2) filter // a) restricted - admin errors.setLength(0); testName = "filters_3_2a"; inputName = "filters_3_2a"; if (null != (output = validateLogstashInput(testName, getTestFile(inputName), errors, false))) { System.out.println("**** FAIL " + testName + ": Should have errored: " + output); } else if (!errors.toString() .startsWith("Security error, non-admin not allowed filter type elasticsearch, allowed options: ")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } //System.out.println("(err="+errors.toString()+")"); // b) restricted - non admin // (USE filters_3_2a for this) errors.setLength(0); testName = "filters_3_2a"; inputName = "filters_3_2a"; if (null == (output = validateLogstashInput(testName, getTestFile(inputName), errors, true))) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } //System.out.println("(val="+output+")"); // c) unrestricted - non admin // (implicitly tested via 3.1bc) // d) no filter at all errors.setLength(0); testName = "filters_3_2d"; inputName = "filters_3_2d"; if (null != (output = validateLogstashInput(testName, getTestFile(inputName), errors, false))) { System.out.println("**** FAIL " + testName + ": Should have errored: " + output); } else if (!errors.toString().startsWith( "Invalid input format, should be 'input { INPUT_TYPE { ... } }' (only one INPUT_TYPE) and also contain a filter, no \"s around them.")) { System.out.println("**** FAIL " + testName + ": " + errors.toString()); } }