Java tutorial
/** * Copyright (c) 2016 Vijay Vijayaram * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software * and associated documentation files (the "Software"), to deal in the Software without restriction, * including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all copies or substantial * portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package com.kumarvv.setl; import com.kumarvv.setl.core.SetlProcessor; import com.kumarvv.setl.model.DS; import com.kumarvv.setl.model.Def; import com.kumarvv.setl.model.Load; import com.kumarvv.setl.model.Status; import com.kumarvv.setl.utils.Chrono; import org.apache.commons.lang3.StringUtils; import org.codehaus.jackson.map.ObjectMapper; import org.pmw.tinylog.Logger; import org.pmw.tinylog.LoggingContext; import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; public class Setl { private static final int STATUS_EVERY = 1000; /** * load definition file * @param filePath * @return */ protected Def loadFile(String filePath) { ObjectMapper mapper = new ObjectMapper(); try { Path path = Paths.get(filePath); final Def def = mapper.readValue(path.toFile(), Def.class); def.setFilePath(path); initCsvPaths(def, path); loadDataStores(def); LoggingContext.put("def", StringUtils.defaultIfBlank(def.getName(), "")); if (def.getFromDS() != null) { Logger.info("fromDS = {}@{}", def.getFromDS().getUsername(), def.getFromDS().getUrl()); } if (def.getToDS() != null) { Logger.info("toDS = {}@{}", def.getToDS().getUsername(), def.getToDS().getUrl()); } return def; } catch (Exception e) { Logger.error("Invalid definition file: {}", e.getMessage()); Logger.trace(e); } return null; } /** * load dataSources from default db.json file when def file skips them * * @param def */ protected void loadDataStores(final Def def) { if (def == null || def.getFilePath() == null) { return; } Path dbPath = def.getFilePath().resolveSibling("db.json"); if (dbPath == null || !dbPath.toFile().exists()) { // resolve from parent folders Path parent = def.getFilePath().getParent(); while (parent != null) { dbPath = parent.resolveSibling("db.json"); if (dbPath != null && dbPath.toFile().exists()) { break; } parent = parent.getParent(); } if (dbPath == null || !dbPath.toFile().exists()) { return; } } try { Def dbDef = new ObjectMapper().readValue(dbPath.toFile(), Def.class); if (def.getFromDS() == null && dbDef.getFromDS() != null) { def.setFromDS(new DS()); def.getFromDS().setUrl(dbDef.getFromDS().getUrl()); def.getFromDS().setUsername(dbDef.getFromDS().getUsername()); def.getFromDS().setPassword(dbDef.getFromDS().getPassword()); } if (def.getToDS() == null && dbDef.getToDS() != null) { def.setToDS(new DS()); def.getToDS().setUrl(dbDef.getToDS().getUrl()); def.getToDS().setUsername(dbDef.getToDS().getUsername()); def.getToDS().setPassword(dbDef.getToDS().getPassword()); } } catch (Exception e) { Logger.error("DB.json error: {}", e.getMessage()); // just ignore } } /** * init csv paths * * @param def * @param defPath */ protected void initCsvPaths(final Def def, Path defPath) { if (def.getExtract().getCsv() != null && StringUtils.isNotEmpty(def.getExtract().getCsv().getFile())) { Path csvPath = defPath.getParent().resolve(def.getExtract().getCsv().getFile()).normalize(); def.getExtract().getCsv().setFilePath(csvPath.toAbsolutePath().toString()); } for (Load load : def.getLoads()) { if (load.getPre() != null && load.getPre().getCsv() != null && StringUtils.isNotEmpty(load.getPre().getCsv().getFile())) { Path csvPath = defPath.getParent().resolve(load.getPre().getCsv().getFile()).normalize(); load.getPre().getCsv().setFilePath(csvPath.toAbsolutePath().toString()); } if (load.getPost() != null && load.getPost().getCsv() != null && StringUtils.isNotEmpty(load.getPost().getCsv().getFile())) { Path csvPath = defPath.getParent().resolve(load.getPost().getCsv().getFile()).normalize(); load.getPost().getCsv().setFilePath(csvPath.toAbsolutePath().toString()); } } } /** * start the process * @param def */ protected void start(final Def def) { if (def == null) { Logger.error("Invalid or blank definition"); return; } LoggingContext.put("defName", def.getName()); LoggingContext.put("loadTable", ""); Logger.info("Processing " + def.getName()); Status status = new Status((s) -> printStatEvery(s)); SetlProcessor processor = new SetlProcessor(status, def); Thread t = new Thread(processor); t.run(); try { t.join(); } catch (InterruptedException ie) { } printStatAll(status); } /** * print stats every * @param status */ protected void printStatEvery(Status status) { if (status.getRowsFound() % STATUS_EVERY == 0) { Logger.info("Found: {}, Processed: {}", status.getRowsFound(), status.getRowsProcessed()); } } /** * print stats * @param status */ protected void printStatAll(Status status) { Logger.info("Found: {}, Processed: {}, failed: {}, inserted: {}, updated: {}", status.getRowsFound(), status.getRowsProcessed(), status.getRowsFailed(), status.getRowsInserted(), status.getRowsUpdated()); } /** * check if valid file * * @param file * @return */ protected String getValidPath(final String file) { if (StringUtils.isEmpty(file)) { return null; } // mac ~ fix String path = file.replaceFirst("^~", System.getProperty("user.home")); File f = Paths.get(path).toFile(); if (!f.exists()) { System.out.println("Invalid file path: " + f.getAbsolutePath()); return null; } LoggingContext.put("file", f.getName()); return f.getAbsolutePath(); } /** * main runner * @param args */ public static void main(String[] args) { LoggingContext.put("def", ""); LoggingContext.put("load", ""); LoggingContext.put("file", ""); if (args == null || args.length == 0) { Logger.error("Definition file (json) is missing"); return; } Chrono kch = Chrono.start("all"); Setl me = new Setl(); for (String file : args) { String filePath = me.getValidPath(file); if (StringUtils.isNotEmpty(filePath)) { Logger.info("Processing definiton: " + file); Def def = me.loadFile(filePath); me.start(def); } } kch.stop(); Logger.info("ALL DONE!"); } }