List of usage examples for org.apache.hadoop.fs FileSystem getLocal
public static LocalFileSystem getLocal(Configuration conf) throws IOException
From source file:com.splout.db.dnode.TestFetcher.java
License:Open Source License
@Test public void testHdfsFetching() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); FileSystem fS = FileSystem.getLocal(conf); SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); Fetcher fetcher = new Fetcher(testConfig); Path path = new Path("tmp-" + TestFetcher.class.getName()); OutputStream oS = fS.create(path); oS.write("This is what happens when you don't know what to write".getBytes()); oS.close();/*ww w. j av a 2 s . co m*/ File f = fetcher.fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString()); assertTrue(f.exists()); assertTrue(f.isDirectory()); File file = new File(f, "tmp-" + TestFetcher.class.getName()); assertTrue(file.exists()); assertEquals("This is what happens when you don't know what to write", Files.toString(file, Charset.defaultCharset())); fS.delete(path, true); FileUtils.deleteDirectory(f); }
From source file:com.splout.db.dnode.TestFetcher.java
License:Open Source License
@Test public void testHdfsFetchingInterrupted() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); final FileSystem fS = FileSystem.getLocal(conf); SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); final Fetcher fetcher = new Fetcher(testConfig); final Path path = new Path("tmp-" + TestFetcher.class.getName()); OutputStream oS = fS.create(path); oS.write("This is what happens when you don't know what to write".getBytes()); oS.close();/*from www . ja v a2s . co m*/ Thread t = new Thread() { @Override public void run() { try { try { File f = fetcher .fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString()); } catch (IOException e) { e.printStackTrace(); } catch (URISyntaxException e) { e.printStackTrace(); } fail("An InterruptedException was expected."); } catch (InterruptedException e) { // Everything good. } } }; // We interrupt the thread before starting so we are sure that the interruption check // will be seen even if the file to copy is very small. t.interrupt(); t.start(); }
From source file:com.splout.db.dnode.TestFetcher.java
License:Open Source License
@Test public void testHdfsFetchingAndThrottling() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); FileSystem fS = FileSystem.getLocal(conf); SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); testConfig.setProperty(FetcherProperties.DOWNLOAD_BUFFER, 4); testConfig.setProperty(FetcherProperties.BYTES_PER_SEC_THROTTLE, 8); Fetcher fetcher = new Fetcher(testConfig); final String str = "This is what happens when you don't know what to write"; Path path = new Path("tmp-" + TestFetcher.class.getName()); OutputStream oS = fS.create(path); oS.write(str.getBytes());/*from w ww . j a v a2s .co m*/ oS.close(); long startTime = System.currentTimeMillis(); File f = fetcher.fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString()); long endTime = System.currentTimeMillis(); double bytesPerSec = (str.getBytes().length / (double) (endTime - startTime)) * 1000; assertEquals(8, bytesPerSec, 0.5); assertTrue(f.exists()); assertTrue(f.isDirectory()); File file = new File(f, "tmp-" + TestFetcher.class.getName()); assertTrue(file.exists()); assertEquals(str, Files.toString(file, Charset.defaultCharset())); fS.delete(path, true); FileUtils.deleteDirectory(f); }
From source file:com.splout.db.engine.RedisOutputFormat.java
License:Apache License
public void initPartition(int partition, Path local) throws IOException { RDBOutputStream rdb = new RDBOutputStream(FileSystem.getLocal(getConf()).create(local)); redisFiles.put(partition, rdb);/* w ww. j a v a 2s. c om*/ rdb.writeHeader(); rdb.writeDatabaseSelector(0); }
From source file:com.splout.db.examples.PageCountsExample.java
License:Apache License
@Override public int run(String[] args) throws Exception { // Validate params etc JCommander jComm = new JCommander(this); jComm.setProgramName("Splout Page Counts example"); try {/*from ww w .j a va 2 s . co m*/ jComm.parse(args); } catch (ParameterException e) { System.err.println(e.getMessage()); jComm.usage(); System.exit(-1); } boolean generate = !noGenerate; // just for clarifying if (generateTupleFiles && deploy) { System.err.println("Can't run a 'dry' TupleFile generation and deploy it."); jComm.usage(); System.exit(-1); } Path outPath = new Path(outputPath); FileSystem outFs = outPath.getFileSystem(getConf()); if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) { File nativeLibs = new File("native"); if (nativeLibs.exists()) { SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf); } } if (generate) { Path inputPath = new Path(this.inputPath); FileSystem inputFileSystem = inputPath.getFileSystem(conf); FileStatus[] fileStatuses = inputFileSystem.listStatus(inputPath); // define the schema that the resultant table will have: date, hour, pagename, pageviews final Schema tableSchema = new Schema("pagecounts", Fields.parse("date:string, hour:string, pagename:string, pageviews:int")); // define the schema of the input files: projectcode, pagename, pageviews, bytes Schema fileSchema = new Schema("pagecountsfile", Fields.parse("projectcode:string, pagename:string, pageviews:int, bytes:long")); // instantiate a TableBuilder TableBuilder tableBuilder = new TableBuilder(tableSchema); // for every input file... for (FileStatus fileStatus : fileStatuses) { String fileName = fileStatus.getPath().getName().toString(); // strip the date and the hour from the file name String fileDate = fileName.split("-")[1]; String fileHour = fileName.split("-")[2].substring(0, 2); // instantiate a custom RecordProcessor to process the records of this file PageCountsRecordProcessor recordProcessor = new PageCountsRecordProcessor(tableSchema, fileDate, fileHour); // use the tableBuilder method for adding each of the files to the mix tableBuilder.addCSVTextFile(fileStatus.getPath(), ' ', TupleTextInputFormat.NO_QUOTE_CHARACTER, TupleTextInputFormat.NO_ESCAPE_CHARACTER, false, false, TupleTextInputFormat.NO_NULL_STRING, fileSchema, recordProcessor); } // partition the dataset by pagename - which should give a fair even distribution. tableBuilder.partitionBy("pagename"); // create a compound index on pagename, date so that typical queries for the dataset will be fast tableBuilder.createIndex("pagename", "date"); long nonExactPageSize = memoryForIndexing / 32000; // number of pages int pageSize = (int) Math.pow(2, (int) Math.round(Math.log(nonExactPageSize) / Math.log(2))); Log.info("Pagesize = " + pageSize + " as memory for indexing was [" + memoryForIndexing + "] and there are 32000 pages."); tableBuilder.initialSQL("pragma page_size=" + pageSize); // insertion order is very important for optimizing query speed because it makes data be co-located in disk tableBuilder.insertionSortOrder(OrderBy.parse("pagename:asc, date:asc")); // instantiate a TablespaceBuilder TablespaceBuilder tablespaceBuilder = new TablespaceBuilder(); // we will partition this dataset in as many partitions as: tablespaceBuilder.setNPartitions(nPartitions); tablespaceBuilder.add(tableBuilder.build()); // we turn a specific SQLite pragma on for making autocomplete queries fast tablespaceBuilder.initStatements("pragma case_sensitive_like=true;"); HadoopUtils.deleteIfExists(outFs, outPath); // finally, instantiate a TablespaceGenerator and execute it TablespaceGenerator tablespaceViewBuilder; if (generateTupleFiles) { // we subclass TablespaceGenerator to be able to run the generation without outputting the SQLite stores, for // benchmark comparisons. // In the future this feature may be useful in general for debugging store creation. tablespaceViewBuilder = new TablespaceGenerator(tablespaceBuilder.build(), outPath, this.getClass()) { @Override public void generateView(Configuration conf, SamplingType samplingType, SamplingOptions samplingOptions) throws Exception { prepareOutput(conf); final int nPartitions = tablespace.getnPartitions(); if (nPartitions > 1) { partitionMap = sample(nPartitions, conf, samplingType, samplingOptions); } else { partitionMap = PartitionMap.oneShardOpenedMap(); } writeOutputMetadata(conf); TupleMRBuilder builder = createMRBuilder(nPartitions, conf); // Set a TupleOutput here instead of SQLiteOutput builder.setOutput(new Path(outputPath, OUT_STORE), new TupleOutputFormat(tableSchema), ITuple.class, NullWritable.class); executeViewGeneration(builder); } }; } else { // ... otherwise a standard TablespaceGenerator is used. tablespaceViewBuilder = new TablespaceGenerator(tablespaceBuilder.build(), outPath, this.getClass()); } tablespaceViewBuilder.generateView(getConf(), SamplingType.FULL_SCAN, new TupleSampler.FullScanSamplingOptions()); } if (deploy) { // use StoreDeployerTool for deploying the already generated dataset StoreDeployerTool deployer = new StoreDeployerTool(qnode, getConf()); ArrayList<TablespaceDepSpec> deployments = new ArrayList<TablespaceDepSpec>(); deployments.add(new TablespaceDepSpec("pagecounts", outPath.toString(), repFactor, null)); deployer.deploy(deployments); } return 1; }
From source file:com.splout.db.hadoop.engine.SploutSQLProxyOutputFormat.java
License:Apache License
@Override public RecordWriter<ITuple, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { long waitTimeHeartBeater = context.getConfiguration().getLong(HeartBeater.WAIT_TIME_CONF, 5000); heartBeater = new HeartBeater(context, waitTimeHeartBeater); heartBeater.needHeartBeat();/* w ww . j a va 2 s . com*/ conf = context.getConfiguration(); this.context = context; outputFormat.setConf(context.getConfiguration()); return new RecordWriter<ITuple, NullWritable>() { // Temporary and permanent Paths for properly writing Hadoop output files private Map<Integer, Path> permPool = new HashMap<Integer, Path>(); private Map<Integer, Path> tempPool = new HashMap<Integer, Path>(); private void initSql(int partition) throws IOException, InterruptedException { // HDFS final location of the generated partition file. It will be // loaded to the temporary folder in the HDFS than finally will be // committed by the OutputCommitter to the proper location. FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter( SploutSQLProxyOutputFormat.this.context); Path perm = new Path(committer.getWorkPath(), partition + ".db"); FileSystem fs = perm.getFileSystem(conf); // Make a task unique name that contains the actual index output name to // make debugging simpler // Note: if using JVM reuse, the sequence number will not be reset for a // new task using the jvm Path temp = conf.getLocalPath("mapred.local.dir", "splout_task_" + SploutSQLProxyOutputFormat.this.context.getTaskAttemptID() + '.' + FILE_SEQUENCE.incrementAndGet()); FileSystem localFileSystem = FileSystem.getLocal(conf); if (localFileSystem.exists(temp)) { localFileSystem.delete(temp, true); } localFileSystem.mkdirs(temp); Path local = fs.startLocalOutput(perm, new Path(temp, partition + ".db")); // permPool.put(partition, perm); tempPool.put(partition, new Path(temp, partition + ".db")); outputFormat.initPartition(partition, local); } @Override public void close(TaskAttemptContext ctx) throws IOException, InterruptedException { FileSystem fs = FileSystem.get(ctx.getConfiguration()); try { if (ctx != null) { heartBeater.setProgress(ctx); } outputFormat.close(); for (Map.Entry<Integer, Path> entry : permPool.entrySet()) { // Hadoop - completeLocalOutput() fs.completeLocalOutput(entry.getValue(), tempPool.get(entry.getKey())); } } finally { // in any case, destroy the HeartBeater heartBeater.cancelHeartBeat(); } } @Override public void write(ITuple tuple, NullWritable ignore) throws IOException, InterruptedException { int partition = (Integer) tuple.get(SploutSQLOutputFormat.PARTITION_TUPLE_FIELD); if (tempPool.get(partition) == null) { initSql(partition); } outputFormat.write(tuple); } }; }
From source file:com.splout.db.hadoop.GeneratorCMD.java
License:Apache License
public int run(String[] args) throws Exception { JCommander jComm = new JCommander(this); jComm.setProgramName(/*from ww w .ja v a 2 s .co m*/ "Splout Tablespaces Generator. Generates tablespaces, ready to be deployed to a Splout Cluster."); try { jComm.parse(args); } catch (Throwable t) { t.printStackTrace(); jComm.usage(); return -1; } if (parallelism < 1) { System.err.println("Parallelism must be greater than 0."); System.exit(1); } log.info("Parsing input parameters..."); // All the tablespaces that will be generated and deployed atomically, hashed by their name // We generate this first so we can detect errors in the configuration before even using Hadoop Map<String, TablespaceSpec> tablespacesToGenerate = new HashMap<String, TablespaceSpec>(); // Partition maps to reuse at indexation. Used when sampling is skipped. final Map<String, PartitionMap> partitionMapsToReuse = new HashMap<String, PartitionMap>(); for (String tablespaceFile : tablespaceFiles) { Path file = new Path(tablespaceFile); FileSystem fS = FileSystem.get(file.toUri(), getConf()); if (!fS.exists(file)) { throw new IllegalArgumentException("Config input file: " + file + " doesn't exist!"); } String strContents = HadoopUtils.fileToString(fS, file); JSONTablespaceDefinition def = JSONSerDe.deSer(strContents, JSONTablespaceDefinition.class); TablespaceSpec spec = def.build(conf); String name = def.getName(); tablespacesToGenerate.put(name, spec); // Reusing partition maps? if (qnodeURL != null) { partitionMapsToReuse.put(name, retrievePartitionMapfromQNode(name)); } } if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) { File nativeLibs = new File("native"); if (nativeLibs.exists()) { SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf); } } Path out = new Path(output); FileSystem outFs = out.getFileSystem(getConf()); HadoopUtils.deleteIfExists(outFs, out); ExecutorService executor = Executors.newFixedThreadPool(parallelism); ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(executor); ArrayList<Future<Boolean>> generatorFutures = new ArrayList<Future<Boolean>>(); // Generate each tablespace for (final Map.Entry<String, TablespaceSpec> tablespace : tablespacesToGenerate.entrySet()) { Path tablespaceOut = new Path(out, tablespace.getKey()); TablespaceSpec spec = tablespace.getValue(); log.info("Generating view with Hadoop (" + tablespace.getKey() + ")"); final TablespaceGenerator viewGenerator = new TablespaceGenerator(spec, tablespaceOut, this.getClass()); generatorFutures.add(ecs.submit(new Callable<Boolean>() { @Override public Boolean call() throws Exception { if (qnodeURL == null) { viewGenerator.generateView(conf, samplingType, new TupleSampler.RandomSamplingOptions()); return true; } else { viewGenerator.generateView(conf, partitionMapsToReuse.get(tablespace.getKey())); return true; } } })); } // Waiting all tasks to finish. for (int i = 0; i < tablespacesToGenerate.size(); i++) { // Get will throw an exception if the callable returned it. try { ecs.take().get(); } catch (ExecutionException e) { // One job was wrong. Stopping the rest. for (Future<Boolean> task : generatorFutures) { task.cancel(true); } executor.shutdown(); throw e; } } executor.shutdown(); log.info("Done!"); return 0; }
From source file:com.splout.db.hadoop.SimpleGeneratorCMD.java
License:Apache License
@Override public int run(String[] args) throws Exception { JCommander jComm = new JCommander(this); jComm.setProgramName("Splout Tablespace View Generator"); try {/* ww w .j a v a2s. c o m*/ jComm.parse(args); } catch (Throwable t) { t.printStackTrace(); jComm.usage(); return -1; } log.info("Parsing input parameters..."); // because Schema is now optional (it can be derived from Cascading or Hive tables), // check that it is provided when using TEXT as input. if (this.inputType.equals(InputType.TEXT) && this.schema == null) { System.err.println("A Pangool Schema must be provided for parsing text files."); jComm.usage(); return -1; } // all input types except Hive require input paths if (!this.inputType.equals(InputType.HIVE) && this.input == null) { System.err.println("Input path must be provided."); jComm.usage(); return -1; } // validate we have all needed args for Cascading integration if (this.inputType.equals(InputType.CASCADING) && this.cascadingColumns == null) { System.err.println( "A comma-separated list of column names must be provided for reading Cascading Tuple files."); jComm.usage(); return -1; } // validate we have all needed args for Hive integration if (this.inputType.equals(InputType.HIVE) && (this.hiveDbName == null || this.hiveTableName == null)) { System.err.println( "When using input type HIVE, both db name and table name must be provided using appropriated arguments."); jComm.usage(); return -1; } TableBuilder tableBuilder; Schema schema = null; if (this.schema != null) { schema = new Schema(tablename, Fields.parse(this.schema)); tableBuilder = new TableBuilder(schema); } else { tableBuilder = new TableBuilder(tablename, conf); } Path out = new Path(output, tablespace); FileSystem outFs = out.getFileSystem(getConf()); HadoopUtils.deleteIfExists(outFs, out); if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) { File nativeLibs = new File("native"); if (nativeLibs.exists()) { SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf); } } int fixedWidth[] = null; if (fixedWidthFields != null) { String[] posStr = fixedWidthFields.split(","); fixedWidth = new int[posStr.length]; for (int i = 0; i < posStr.length; i++) { try { fixedWidth[i] = new Integer(posStr[i]); } catch (NumberFormatException e) { System.err.println("Wrongly formed fixed with field list: [" + fixedWidthFields + "]. " + posStr[i] + " does not look as a number."); return -1; } } } TablespaceBuilder builder = new TablespaceBuilder(); builder.setEngineClassName(engine); log.info("Using engine: " + engine); if (inputType.equals(InputType.TEXT)) { Path inputPath = new Path(input); if (fixedWidth == null) { // CSV tableBuilder.addCSVTextFile(inputPath, separator, quotes, escape, skipHeading, strictQuotes, nullString); } else { // Fixed Width tableBuilder.addFixedWidthTextFile(inputPath, schema, fixedWidth, skipHeading, nullString, null); } } else if (inputType.equals(InputType.TUPLE)) { // Pangool Tuple file Path inputPath = new Path(input); if (schema != null) { tableBuilder.addTupleFile(inputPath, schema); } else { tableBuilder.addTupleFile(inputPath); } } else if (inputType.equals(InputType.CASCADING)) { // Cascading Tuple file Path inputPath = new Path(input); tableBuilder.addCascadingTable(inputPath, cascadingColumns.split(",")); } else if (inputType.equals(InputType.HIVE)) { // Hive table tableBuilder.addHiveTable(hiveDbName, hiveTableName); } String[] strPartitionByFields = this.partitionByFields.split(","); tableBuilder.partitionBy(strPartitionByFields); for (String strIndex : indexes) { tableBuilder.createIndex(strIndex.split(",")); } log.info("Generating view with Hadoop..."); builder.add(tableBuilder.build()); builder.setNPartitions(nPartitions); TablespaceGenerator viewGenerator = new TablespaceGenerator(builder.build(), out, this.getClass()); viewGenerator.generateView(conf, samplingType, new TupleSampler.RandomSamplingOptions()); log.info("Success! Tablespace [" + tablespace + "] with table [" + tablename + "] properly created at path [" + out + "]"); return 0; }
From source file:com.splout.db.hadoop.TestTablespaceGeneratorJavaScript.java
License:Apache License
public void simpleTest(SamplingType samplingType) throws Exception { Runtime.getRuntime().exec("rm -rf " + INPUT); Runtime.getRuntime().exec("rm -rf " + OUTPUT); Configuration conf = new Configuration(); TupleFile.Writer writer = new TupleFile.Writer(FileSystem.get(conf), conf, new Path(INPUT), theSchema1); writer.append(TestTablespaceGenerator.getTuple("aa1", "value1")); writer.append(TestTablespaceGenerator.getTuple("aa2", "value2")); writer.append(TestTablespaceGenerator.getTuple("ab1", "value3")); writer.append(TestTablespaceGenerator.getTuple("ab2", "value4")); writer.append(TestTablespaceGenerator.getTuple("bb1", "value5")); writer.append(TestTablespaceGenerator.getTuple("bb2", "value6")); writer.close();//w w w . ja v a 2 s . c o m TablespaceBuilder builder = new TablespaceBuilder(); builder.setNPartitions(3); TableBuilder tableBuilder = new TableBuilder(theSchema1); tableBuilder.addFile(new TableInput(new TupleInputFormat(), new HashMap<String, String>(), theSchema1, new IdentityRecordProcessor(), new Path(INPUT))); // Partition by a javascript that returns the first two characters tableBuilder.partitionByJavaScript( "function partition(record) { var str = record.get('id').toString(); return str.substring(0, 2); }"); builder.add(tableBuilder.build()); TablespaceGenerator viewGenerator = new TablespaceGenerator(builder.build(), new Path(OUTPUT), this.getClass()); viewGenerator.generateView(conf, samplingType, new TupleSampler.RandomSamplingOptions()); PartitionMap partitionMap = JSONSerDe.deSer( HadoopUtils.fileToString(FileSystem.getLocal(conf), new Path(OUTPUT, "partition-map")), PartitionMap.class); assertEquals(null, partitionMap.getPartitionEntries().get(0).getMin()); assertEquals("aa", partitionMap.getPartitionEntries().get(0).getMax()); assertEquals("aa", partitionMap.getPartitionEntries().get(1).getMin()); assertEquals("ab", partitionMap.getPartitionEntries().get(1).getMax()); assertEquals("ab", partitionMap.getPartitionEntries().get(2).getMin()); assertEquals(null, partitionMap.getPartitionEntries().get(2).getMax()); Runtime.getRuntime().exec("rm -rf " + INPUT); Runtime.getRuntime().exec("rm -rf " + OUTPUT); }
From source file:com.splout.db.integration.HadoopIntegrationTest.java
License:Apache License
@Override public int run(String[] args) throws Exception { // Validate params etc JCommander jComm = new JCommander(this); jComm.setProgramName("Splout Hadoop Compatibility Integration Test"); try {/* www . ja v a2s .c om*/ jComm.parse(args); } catch (ParameterException e) { System.err.println(e.getMessage()); jComm.usage(); System.exit(-1); } Path tmpHdfsPath = new Path( "tmp-" + HadoopIntegrationTest.class.getName() + "-" + System.currentTimeMillis()); FileSystem fS = tmpHdfsPath.getFileSystem(getConf()); fS.mkdirs(tmpHdfsPath); fS.mkdirs(new Path(tmpHdfsPath, "input")); fS.mkdirs(new Path(tmpHdfsPath, "output")); boolean isLocal = FileSystem.get(conf).equals(FileSystem.getLocal(conf)); if (!isLocal) { SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf); } tmpHdfsPath = tmpHdfsPath.makeQualified(fS); Path pageCounts = new Path(input); FileUtil.copy(FileSystem.getLocal(getConf()), pageCounts, fS, new Path(tmpHdfsPath, "input"), false, getConf()); SimpleGeneratorCMD generator = new SimpleGeneratorCMD(); generator.setConf(getConf()); if (generator.run(new String[] { "-tb", "pagecountsintegration", "-t", "pagecounts", "-i", tmpHdfsPath + "/input", "-o", tmpHdfsPath + "/output", "-s", "projectcode:string, pagename:string, visits:int, bytes:long", "-pby", "projectcode,pagename", "-sep", "\" \"", "-p", "2", "-e", engine }) < 0) { throw new RuntimeException("Generator failed!"); } SploutClient client = new SploutClient(qnode); QNodeStatus status = client.overview(); long previousVersion = -1; if (status.getTablespaceMap().get("pagecountsintegration") != null) { previousVersion = status.getTablespaceMap().get("pagecountsintegration").getVersion(); } DeployerCMD deployer = new DeployerCMD(); deployer.setConf(getConf()); if (deployer.run(new String[] { "-r", "2", "-q", qnode, "-root", tmpHdfsPath + "/output", "-ts", "pagecountsintegration" }) < 0) { throw new RuntimeException("Deployer failed!"); } long waitedSoFar = 0; status = client.overview(); while (status.getTablespaceMap().get("pagecountsintegration") == null || previousVersion == status.getTablespaceMap().get("pagecountsintegration").getVersion()) { Thread.sleep(2000); waitedSoFar += 2000; status = client.overview(); if (waitedSoFar > 90000) { throw new RuntimeException( "Deploy must have failed in Splout's server. Waiting too much for it to complete."); } } previousVersion = status.getTablespaceMap().get("pagecountsintegration").getVersion(); QueryStatus qStatus = client.query("pagecountsintegration", "*", "SELECT * FROM pagecounts;", null); System.out.println(qStatus.getResult()); if (qStatus.getResult() == null) { throw new RuntimeException("Something failed as query() is returning null!"); } System.out.println("Everything fine."); return 1; }