Java tutorial
/** * Copyright (c) 2016 Modus Operandi, Inc. * * This is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, either version 3 of the License, or any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more * details. A copy of the GNU Lesser General Public License is distributed along * with this program and can be found at * <http://www.gnu.org/licenses/lgpl.html>. */ package com.modusoperandi.dragonfly.widgets.fileQuery; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.FilenameFilter; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import javax.servlet.http.HttpServletRequest; import org.apache.wicket.ajax.AjaxRequestTarget; import org.apache.wicket.ajax.form.OnChangeAjaxBehavior; import org.apache.wicket.ajax.markup.html.form.AjaxButton; import org.apache.wicket.markup.html.WebPage; import org.apache.wicket.markup.html.basic.Label; import org.apache.wicket.markup.html.form.DropDownChoice; import org.apache.wicket.markup.html.form.Form; import org.apache.wicket.markup.html.form.TextField; import org.apache.wicket.model.PropertyModel; import org.apache.wicket.request.cycle.RequestCycle; import org.apache.wicket.util.time.Time; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; import static org.elasticsearch.node.NodeBuilder.nodeBuilder; /** * The Class */ public class FileQueryWidgetPage extends WebPage { /** * The Class DeleteIndexAjaxButton deletes a specified Elasticsearch index. */ private class DeleteIndexAjaxButton extends AjaxButton { private static final long serialVersionUID = -4476024675011822240L; /** * Instantiates a new delete index ajax button. */ public DeleteIndexAjaxButton() { super("deleteButton"); } /* * (non-Javadoc) * * @see org.apache.wicket.ajax.markup.html.form.AjaxButton#onSubmit(org.apache.wicket.ajax.AjaxRequestTarget, * org.apache.wicket.markup.html.form.Form) */ @Override protected void onSubmit(final AjaxRequestTarget target, final Form<?> form) { if (getEsClient().admin().indices().prepareExists(indexName.toLowerCase()).execute().actionGet() .isExists()) { getEsClient().admin().indices().prepareDelete(indexName.toLowerCase()).execute().actionGet(); } status = "Deleted"; target.add(statusField); } } /** * The Class QueryAjaxButton submits a DIB query and stores the result in an Elasticsearch index. */ private class QueryAjaxButton extends AjaxButton { private static final long serialVersionUID = -4476024675011822240L; /** * Instantiates a new query ajax button. */ public QueryAjaxButton() { super("okButton"); } /* * (non-Javadoc) * * @see org.apache.wicket.ajax.markup.html.form.AjaxButton#onSubmit(org.apache.wicket.ajax.AjaxRequestTarget, * org.apache.wicket.markup.html.form.Form) */ @Override protected void onSubmit(final AjaxRequestTarget target, final Form<?> form) { insertFileIntoElasticSearch(); status = "Data inserted"; target.add(statusField); } } /** * The Class ModalUpdateOnlyOnChangeAjaxBehavior. * * An ajax behavior that does nothing other than update the modal associated with the control that has this behavior * */ protected static class ModalUpdateOnlyOnChangeAjaxBehavior extends OnChangeAjaxBehavior { private static final long serialVersionUID = -8237567125967702536L; @Override protected void onUpdate(final AjaxRequestTarget target) { // Wicket will update the model based on changes } } /** * The Class ListFilenameFilter. * * Filters files when listing directories. * */ static class ListFilenameFilter implements FilenameFilter { private final List<String> extensions; /** * Instantiates a new list filename filter. * * @param extensions * the extensions to filter for */ public ListFilenameFilter(final List<String> extensions) { this.extensions = extensions; } /* * (non-Javadoc) * * @see java.io.FilenameFilter#accept(java.io.File, java.lang.String) * * Filters by accepting files with "type" for a file extension */ @Override public boolean accept(final File dir, final String name) { return extensions.stream().anyMatch((i) -> (name.toLowerCase().endsWith(i.toLowerCase()))); } } private transient static Client esClient = null; private transient static Node esNode = null; private static final Logger LOGGER = Logger.getLogger(FileQueryWidgetPage.class.getName()); private static final int ONE_DAY_IN_SECONDS = 86400; private static final long serialVersionUID = 1L; private static final String DATA_DIR = "dragonfly/"; /** * Csv to list. * * This converts a comma separated value list to a list of strings Any "," or "\" that should appear in the data * results (i.e. is not a separator or escape char) should be preceded by a "\" (@see listToCsv) * * @param csv * the csv created by or in comformance with listToCsv() * @param stingList * a list of string to encode */ public static void csvToList(final String csv, final List<String> stingList) { byte[] headerCsv; try { headerCsv = csv.getBytes("UTF-8"); final StringBuffer buffer = new StringBuffer(); boolean insideQuotes = false; for (int i = 0; i < headerCsv.length; i++) { if (headerCsv[i] == '\\') { // Next char was escaped i++; if (i < headerCsv.length) { buffer.append((char) headerCsv[i]); } } else if (headerCsv[i] == '\"') { insideQuotes = !insideQuotes; } else if (!insideQuotes && (headerCsv[i] == ',')) { // Seperator stingList.add(buffer.toString()); buffer.setLength(0); } else { buffer.append((char) headerCsv[i]); } } stingList.add(buffer.toString()); } catch (final UnsupportedEncodingException e) { LOGGER.log(Level.SEVERE, "Failed to convert csv to list.", e); } } public static void onDestroy() { if (esClient != null) { LOGGER.info("Closing client"); esClient.close(); esClient = null; } if (esNode != null) { LOGGER.info("Closing node"); esNode.close(); esNode = null; } } private static Client getEsClient() { if (esClient == null) { LOGGER.info("FileQueryWidget creating node"); // //////////// // Local final Settings settings = Settings.settingsBuilder() .put("node.name", FileQueryWidgetPage.class.getSimpleName()).put("node.data", true) .put("node.master", false).put("http.enabled", false).put("path.home", ".") .put("index.store.type", "simplefs").put("index.number_of_shards", 1) .put("index.number_of_replicas", 0).put("index.shard.check_on_startup", "fix") .put("path.data", "data/DRAGONFLY/" + FileQueryWidgetPage.class.getSimpleName()) .put("script.inline", true).put("script.indexed", true) // .put("discovery.zen.ping.multicast.enabled", false) // .put("discovery.zen.ping.unicast.enabled", true) // .put("discovery.zen.ping.unicast.hosts", "[localhost:9300]") .build(); esNode = nodeBuilder().settings(settings).clusterName("DRAGONFLY").node(); esClient = esNode.client(); } return esClient; } /** * Gets a file list of the specified type from the DATA_DIR directory. * * @param fileNames * the file names * @param types * @param defaultValue */ protected static void getFileList(final List<String> fileNames, final List<String> types, final String defaultValue) { final File dir = new File(DATA_DIR); final ListFilenameFilter filter = new ListFilenameFilter(types); final String[] children = dir.list(filter); System.out.println(dir.getAbsolutePath()); fileNames.clear(); if (children != null) { fileNames.addAll(Arrays.asList(children)); // Get filename of file or directory } if (defaultValue != null) { fileNames.add(0, defaultValue); } } private final List<String> fileList = new LinkedList<>(); private String fileName; private String fileType; private String indexName; private final List<String> mappingFileList = new LinkedList<>(); private String mappingFileName; private final DropDownChoice<?> selectFile; private final DropDownChoice<?> selectMappingFile; private String status = ""; private final Label statusField; private String typeName; /** * Instantiates a new file query widget page. */ public FileQueryWidgetPage() { final Form<?> queryForm = new Form<>("queryForm"); // //////////////////////////// // File // Populate the drop down list with the files on the server getFileList(fileList, Arrays.asList(".csv"), null); selectFile = new DropDownChoice<>("fileName", new PropertyModel<String>(this, "fileName"), fileList); // Enable the transform whenever the user selects one selectFile.add(new OnChangeAjaxBehavior() { private static final long serialVersionUID = 2981822623630720652L; @Override protected void onUpdate(final AjaxRequestTarget target) { getFileList(fileList, Arrays.asList(".csv"), null); target.add(selectFile); status = "Update pending"; target.add(statusField); } }); queryForm.add(selectFile); // //////////////////////////// // Mapping File // Populate the drop down list with the files on the server getFileList(mappingFileList, Arrays.asList(".mapping"), null); selectMappingFile = new DropDownChoice<>("mappingFileName", new PropertyModel<String>(this, "mappingFileName"), mappingFileList); // Enable the transform whenever the user selects one selectMappingFile.add(new OnChangeAjaxBehavior() { private static final long serialVersionUID = 2981822623630720652L; @Override protected void onUpdate(final AjaxRequestTarget target) { getFileList(mappingFileList, Arrays.asList(".mapping"), null); target.add(selectMappingFile); status = "Update pending"; target.add(statusField); } }); queryForm.add(selectMappingFile); // //////////////////////////// // File Type final DropDownChoice<?> selectType = new DropDownChoice<>("fileType", new PropertyModel<String>(this, "fileType"), Arrays.asList("CSV")); // Enable the transform whenever the user selects one selectType.add(new OnChangeAjaxBehavior() { private static final long serialVersionUID = 2981822623630720652L; @Override protected void onUpdate(final AjaxRequestTarget target) { status = "Update pending"; target.add(statusField); } }); queryForm.add(selectType); // //////////////////////////// // Query Name final TextField<String> queryName = new TextField<>("dataSetName", new PropertyModel<>(this, "indexName")); queryName.add(new ModalUpdateOnlyOnChangeAjaxBehavior()); queryForm.add(queryName); add(queryForm); // //////////////////////////// // Status statusField = new Label("status", new PropertyModel<>(this, "status")); statusField.setOutputMarkupId(true); queryForm.add(statusField); // //////////////////////////// // Add the Query and Delete buttons to the form. queryForm.add(new QueryAjaxButton()); queryForm.add(new DeleteIndexAjaxButton()); } private void insertCsvFileIntoElasticSearch() { BufferedReader fileReader = null; try { String line; // Create the file reader fileReader = new BufferedReader(new FileReader(DATA_DIR + fileName)); // Read the file line by line boolean firstLine = true; final List<String> columnHeaders = new LinkedList<>(); while ((line = fileReader.readLine()) != null) { if (firstLine) { firstLine = false; csvToList(line, columnHeaders); } else { // Get all tokens available in line final List<String> tokens = new LinkedList<>(); csvToList(line, tokens); final Map<String, Object> data = new HashMap<>(); for (int i = 0; (i < columnHeaders.size()) && (i < tokens.size()); i++) { data.put(columnHeaders.get(i), tokens.get(i)); } try { getEsClient().prepareIndex(indexName.toLowerCase(), typeName).setSource(data).execute() .actionGet(); } catch (final Exception e) { LOGGER.log(Level.SEVERE, "Failed to insert data into ES: " + line, e); } } } try { fileReader.close(); } catch (final IOException e1) { LOGGER.log(Level.SEVERE, "Failed to close fileReader following successful insertion into ES.", e1); } } catch (final IOException e) { LOGGER.log(Level.SEVERE, "Failed to insert into ES.", e); if (fileReader != null) { try { fileReader.close(); } catch (final IOException e1) { LOGGER.log(Level.SEVERE, "Failed to close fileReader.", e1); } } } } private void insertFileIntoElasticSearch() { if ((mappingFileName != null)) { insertMappingIntoElasticSearch(); } if ((fileType != null) && (fileName != null) && (indexName != null)) { if (fileType.equals("CSV")) { insertCsvFileIntoElasticSearch(); } } } private void insertMappingIntoElasticSearch() { BufferedReader br = null; final StringBuilder mapping = new StringBuilder(); try { String currentLine; br = new BufferedReader(new FileReader(DATA_DIR + mappingFileName)); while ((currentLine = br.readLine()) != null) { mapping.append(currentLine); } } catch (final IOException e) { LOGGER.log(Level.SEVERE, "Failed to append currentLine to mapping.", e); } finally { try { if (br != null) { br.close(); } } catch (final IOException ex) { LOGGER.log(Level.SEVERE, "Failed to close BufferedReader.", ex); } } typeName = mappingFileName.substring(0, mappingFileName.indexOf('.')); if (getEsClient().admin().indices().prepareExists(indexName.toLowerCase()).execute().actionGet() .isExists()) { // getEsClient().admin().indices().prepareDelete(indexName.toLowerCase()).execute().actionGet(); } if (!getEsClient().admin().indices().prepareExists(indexName.toLowerCase()).execute().actionGet() .isExists()) { getEsClient().admin().indices().prepareCreate(indexName.toLowerCase()).execute().actionGet(); } getEsClient().admin().indices().preparePutMapping().setIndices(indexName.toLowerCase()).setType(typeName) .setSource(mapping.toString()).execute().actionGet(); } /* * (non-Javadoc) * * @see org.apache.wicket.markup.html.WebPage#onAfterRender() */ @Override protected void onAfterRender() { super.onAfterRender(); // Increase the HTTP session timeout to 1 day ((HttpServletRequest) RequestCycle.get().getRequest().getContainerRequest()).getSession() .setMaxInactiveInterval(ONE_DAY_IN_SECONDS); } protected @Override void setHeaders(final org.apache.wicket.request.http.WebResponse response) { super.setHeaders(response); response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate"); // HTTP 1.1. response.setHeader("Pragma", "no-cache"); // HTTP 1.0. response.setDateHeader("Expires", Time.START_OF_UNIX_TIME); // Proxies. } }