List of usage examples for org.apache.hadoop.fs Path suffix
public Path suffix(String suffix)
From source file:fm.last.feathers.output.IndexedPlainTypedBytes.java
License:Apache License
@Override public RecordWriter<TypedBytesWritable, TypedBytesWritable> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { Path file = FileOutputFormat.getTaskOutputPath(job, name); Path index = file.suffix("-index"); FileSystem fs = file.getFileSystem(job); FSDataOutputStream fileOut = fs.create(file, progress); FSDataOutputStream indexOut = fs.create(index); return new IndexingPairWriter(fileOut, indexOut); }
From source file:gobblin.data.management.trash.Trash.java
License:Apache License
/** * Move a path to trash. The absolute path of the input path will be replicated under the trash directory. * @param path {@link org.apache.hadoop.fs.FileSystem} path to move to trash. * @return true if move to trash was done successfully. * @throws IOException//from w ww.java 2 s . c om */ @Override public boolean moveToTrash(Path path) throws IOException { Path fullyResolvedPath = path.isAbsolute() ? path : new Path(this.fs.getWorkingDirectory(), path); Path targetPathInTrash = PathUtils.mergePaths(this.trashLocation, fullyResolvedPath); if (!this.fs.exists(targetPathInTrash.getParent())) { this.fs.mkdirs(targetPathInTrash.getParent()); } else if (this.fs.exists(targetPathInTrash)) { targetPathInTrash = targetPathInTrash.suffix("_" + System.currentTimeMillis()); } return this.fs.rename(fullyResolvedPath, targetPathInTrash); }
From source file:io.prestosql.plugin.hive.AbstractTestHiveClient.java
License:Apache License
@Test public void testTableCreationIgnoreExisting() { List<Column> columns = ImmutableList .of(new Column("dummy", HiveType.valueOf("uniontype<smallint,tinyint>"), Optional.empty())); SchemaTableName schemaTableName = temporaryTable("create"); ConnectorSession session = newSession(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); PrincipalPrivileges privileges = testingPrincipalPrivilege(session); Path targetPath; try {/*w w w. j av a 2 s . c o m*/ try (Transaction transaction = newTransaction()) { LocationService locationService = getLocationService(); LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); Table table = createSimpleTable(schemaTableName, columns, session, targetPath, "q1"); transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), false, EMPTY_TABLE_STATISTICS); Optional<Table> tableHandle = transaction.getMetastore(schemaName).getTable(schemaName, tableName); assertTrue(tableHandle.isPresent()); transaction.commit(); } // try creating it again from another transaction with ignoreExisting=false try (Transaction transaction = newTransaction()) { Table table = createSimpleTable(schemaTableName, columns, session, targetPath.suffix("_2"), "q2"); transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), false, EMPTY_TABLE_STATISTICS); transaction.commit(); fail("Expected exception"); } catch (PrestoException e) { assertInstanceOf(e, TableAlreadyExistsException.class); } // try creating it again from another transaction with ignoreExisting=true try (Transaction transaction = newTransaction()) { Table table = createSimpleTable(schemaTableName, columns, session, targetPath.suffix("_3"), "q3"); transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), true, EMPTY_TABLE_STATISTICS); transaction.commit(); } // at this point the table should exist, now try creating the table again with a different table definition columns = ImmutableList.of(new Column("new_column", HiveType.valueOf("string"), Optional.empty())); try (Transaction transaction = newTransaction()) { Table table = createSimpleTable(schemaTableName, columns, session, targetPath.suffix("_4"), "q4"); transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), true, EMPTY_TABLE_STATISTICS); transaction.commit(); fail("Expected exception"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), TRANSACTION_CONFLICT.toErrorCode()); assertEquals(e.getMessage(), format("Table already exists with a different schema: '%s'", schemaTableName.getTableName())); } } finally { dropTable(schemaTableName); } }
From source file:org.apache.accumulo.core.client.mock.MockTableOperations.java
License:Apache License
@Override public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { long time = System.currentTimeMillis(); MockTable table = acu.tables.get(tableName); if (table == null) { throw new TableNotFoundException(null, tableName, "The table was not found"); }/*from w w w .ja v a 2 s . co m*/ Path importPath = new Path(dir); Path failurePath = new Path(failureDir); FileSystem fs = acu.getFileSystem(); /* * check preconditions */ // directories are directories if (fs.isFile(importPath)) { throw new IOException("Import path must be a directory."); } if (fs.isFile(failurePath)) { throw new IOException("Failure path must be a directory."); } // failures are writable Path createPath = failurePath.suffix("/.createFile"); FSDataOutputStream createStream = null; try { createStream = fs.create(createPath); } catch (IOException e) { throw new IOException("Error path is not writable."); } finally { if (createStream != null) { createStream.close(); } } fs.delete(createPath, false); // failures are empty FileStatus[] failureChildStats = fs.listStatus(failurePath); if (failureChildStats.length > 0) { throw new IOException("Error path must be empty."); } /* * Begin the import - iterate the files in the path */ for (FileStatus importStatus : fs.listStatus(importPath)) { try { FileSKVIterator importIterator = FileOperations.getInstance().newReaderBuilder() .forFile(importStatus.getPath().toString(), fs, fs.getConf()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).seekToBeginning() .build(); while (importIterator.hasTop()) { Key key = importIterator.getTopKey(); Value value = importIterator.getTopValue(); if (setTime) { key.setTimestamp(time); } Mutation mutation = new Mutation(key.getRow()); if (!key.isDeleted()) { mutation.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp(), value); } else { mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp()); } table.addMutation(mutation); importIterator.next(); } } catch (Exception e) { FSDataOutputStream failureWriter = null; DataInputStream failureReader = null; try { failureWriter = fs.create(failurePath.suffix("/" + importStatus.getPath().getName())); failureReader = fs.open(importStatus.getPath()); int read = 0; byte[] buffer = new byte[1024]; while (-1 != (read = failureReader.read(buffer))) { failureWriter.write(buffer, 0, read); } } finally { if (failureReader != null) failureReader.close(); if (failureWriter != null) failureWriter.close(); } } fs.delete(importStatus.getPath(), true); } }
From source file:org.apache.accumulo.core.client.mock.MockTableOperationsImpl.java
License:Apache License
@Override public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { long time = System.currentTimeMillis(); MockTable table = acu.tables.get(tableName); if (table == null) { throw new TableNotFoundException(null, tableName, "The table was not found"); }// www .j av a 2s. c o m Path importPath = new Path(dir); Path failurePath = new Path(failureDir); FileSystem fs = acu.getFileSystem(); /* * check preconditions */ // directories are directories if (fs.isFile(importPath)) { throw new IOException("Import path must be a directory."); } if (fs.isFile(failurePath)) { throw new IOException("Failure path must be a directory."); } // failures are writable Path createPath = failurePath.suffix("/.createFile"); FSDataOutputStream createStream = null; try { createStream = fs.create(createPath); } catch (IOException e) { throw new IOException("Error path is not writable."); } finally { if (createStream != null) { createStream.close(); } } fs.delete(createPath, false); // failures are empty FileStatus[] failureChildStats = fs.listStatus(failurePath); if (failureChildStats.length > 0) { throw new IOException("Error path must be empty."); } /* * Begin the import - iterate the files in the path */ for (FileStatus importStatus : fs.listStatus(importPath)) { try { FileSKVIterator importIterator = FileOperations.getInstance().openReader( importStatus.getPath().toString(), true, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration()); while (importIterator.hasTop()) { Key key = importIterator.getTopKey(); Value value = importIterator.getTopValue(); if (setTime) { key.setTimestamp(time); } Mutation mutation = new Mutation(key.getRow()); if (!key.isDeleted()) { mutation.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp(), value); } else { mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibilityData().toArray()), key.getTimestamp()); } table.addMutation(mutation); importIterator.next(); } } catch (Exception e) { FSDataOutputStream failureWriter = null; DataInputStream failureReader = null; try { failureWriter = fs.create(failurePath.suffix("/" + importStatus.getPath().getName())); failureReader = fs.open(importStatus.getPath()); int read = 0; byte[] buffer = new byte[1024]; while (-1 != (read = failureReader.read(buffer))) { failureWriter.write(buffer, 0, read); } } finally { if (failureReader != null) failureReader.close(); if (failureWriter != null) failureWriter.close(); } } fs.delete(importStatus.getPath(), true); } }
From source file:org.apache.avro.mapreduce.TestAvroMultipleOutputs.java
License:Apache License
@Test public void testAvroGenericOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(GenericStatsReducer.class); AvroJob.setOutputKeySchema(job, STATS_SCHEMA); AvroMultipleOutputs.addNamedOutput(job, "myavro", AvroKeyOutputFormat.class, STATS_SCHEMA, null); AvroMultipleOutputs.addNamedOutput(job, "myavro1", AvroKeyOutputFormat.class, STATS_SCHEMA_2); job.setOutputFormatClass(AvroKeyOutputFormat.class); String dir = System.getProperty("test.dir", ".") + "/mapred"; Path outputPath = new Path(dir + "/out"); outputPath.getFileSystem(job.getConfiguration()).delete(outputPath); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); DataFileReader<GenericData.Record> reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); Map<String, Integer> counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); }/*from w w w .jav a2s . co m*/ reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro1-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA_2)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name1")).toString(), (Integer) record.get("count1")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/testnewwrite-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/testnewwrite2-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA_2)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name1")).toString(), (Integer) record.get("count1")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/testwritenonschema-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); }
From source file:org.apache.avro.mapreduce.TestAvroMultipleOutputs.java
License:Apache License
@Test public void testAvroSpecificOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); AvroMultipleOutputs.addNamedOutput(job, "myavro3", AvroKeyOutputFormat.class, TextStats.SCHEMA$, null); job.setReducerClass(SpecificStatsReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); String dir = System.getProperty("test.dir", ".") + "/mapred"; Path outputPath = new Path(dir + "/out-specific"); outputPath.getFileSystem(job.getConfiguration()).delete(outputPath); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro3-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); }/*ww w . ja va 2 s .c o m*/ reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); }
From source file:org.apache.avro.mapreduce.TestAvroMultipleOutputs.java
License:Apache License
@Test public void testAvroInput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.avro").toURI().toString())); job.setInputFormatClass(AvroKeyInputFormat.class); AvroJob.setInputKeySchema(job, TextStats.SCHEMA$); AvroMultipleOutputs.addNamedOutput(job, "myavro3", AvroKeyOutputFormat.class, TextStats.SCHEMA$, null); job.setMapperClass(StatCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(SpecificStatsReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-specific-input"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro3-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); }//from w w w. j a v a2 s . com reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); }
From source file:org.apache.avro.mapreduce.TestAvroMultipleOutputs.java
License:Apache License
@Test public void testAvroMapOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.avro").toURI().toString())); job.setInputFormatClass(AvroKeyInputFormat.class); AvroJob.setInputKeySchema(job, TextStats.SCHEMA$); job.setMapperClass(SortMapper.class); AvroJob.setMapOutputKeySchema(job, TextStats.SCHEMA$); job.setMapOutputValueClass(NullWritable.class); job.setReducerClass(SortReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-specific-input"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); }// www . j ava2s . c om reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); }
From source file:org.apache.avro.mapreduce.TestWordCount.java
License:Apache License
@Test public void testAvroGenericOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(GenericStatsReducer.class); AvroJob.setOutputKeySchema(job, STATS_SCHEMA); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-generic"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<GenericData.Record> reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); Map<String, Integer> counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); }//from w w w . j ava 2s. c o m reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); }