List of usage examples for java.nio.channels FileChannel open
public static FileChannel open(Path path, OpenOption... options) throws IOException
From source file:com.spectralogic.ds3cli.command.PutObject.java
private void transfer(final Ds3ClientHelpers helpers, final Ds3Object ds3Obj) throws IOException, XmlProcessingException { final WriteJobOptions writeJobOptions = WriteJobOptions.create(); writeJobOptions.setForce(force);/*from w w w . j a v a 2s . c om*/ if (priority != null) { writeJobOptions.withPriority(priority); } final Ds3ClientHelpers.Job putJob = helpers .startWriteJob(this.bucketName, Lists.newArrayList(ds3Obj), writeJobOptions) .withMaxParallelRequests(this.numberOfThreads); if (!Guard.isMapNullOrEmpty(metadata)) { putJob.withMetadata(new MetadataAccess() { @Override public Map<String, String> getMetadataValue(final String s) { return new ImmutableMap.Builder<String, String>() .putAll(MetadataUtils.getMetadataValues(objectPath)).putAll(metadata).build(); } }); } putJob.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() { @Override public SeekableByteChannel buildChannel(final String s) throws IOException { return FileChannel.open(objectPath, StandardOpenOption.READ); } }); }
From source file:com.github.jinahya.verbose.codec.BinaryCodecTest.java
protected final void encodeDecode(final Path expectedPath) throws IOException { if (expectedPath == null) { throw new NullPointerException("null expectedPath"); }/*ww w. j av a2 s . c o m*/ try (final FileChannel expectedChannel = FileChannel.open(expectedPath, StandardOpenOption.READ)) { encodeDecode(expectedChannel); } }
From source file:com.github.jinahya.verbose.codec.BinaryCodecTest.java
@Test(enabled = true, invocationCount = 128) public void encodeDecodeRandomFiles() throws IOException { final Path expectedPath = Files.createTempFile("test", null); getRuntime().addShutdownHook(new Thread(() -> { try {//from w ww .j a va 2s . c o m Files.delete(expectedPath); } catch (final IOException ioe) { ioe.printStackTrace(System.err); } })); final byte[] array = new byte[1024]; final ByteBuffer buffer = ByteBuffer.wrap(array); try (final FileChannel decodedChannel = FileChannel.open(expectedPath, StandardOpenOption.WRITE)) { final int count = current().nextInt(128); for (int i = 0; i < count; i++) { current().nextBytes(array); decodedChannel.write(buffer); } decodedChannel.force(false); } encodeDecode(expectedPath); }
From source file:io.pravega.segmentstore.storage.impl.extendeds3.S3FileSystemImpl.java
@Synchronized @Override//from w w w .j a v a 2s . co m public CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request) { Map<Integer, CopyPartRequest> partMap = multipartUploads.get(request.getKey()); if (partMap == null) { throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", ""); } try { partMap.forEach((index, copyPart) -> { if (copyPart.getKey() != copyPart.getSourceKey()) { Path sourcePath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getSourceKey()); Path targetPath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getKey()); try (FileChannel sourceChannel = FileChannel.open(sourcePath, StandardOpenOption.READ); FileChannel targetChannel = FileChannel.open(targetPath, StandardOpenOption.WRITE)) { targetChannel.transferFrom(sourceChannel, Files.size(targetPath), copyPart.getSourceRange().getLast() + 1 - copyPart.getSourceRange().getFirst()); targetChannel.close(); AclSize aclMap = this.aclMap.get(copyPart.getKey()); this.aclMap.put(copyPart.getKey(), aclMap.withSize(Files.size(targetPath))); } catch (IOException e) { throw new S3Exception("NoSuchKey", 404, "NoSuchKey", ""); } } }); } finally { multipartUploads.remove(request.getKey()); } return new CompleteMultipartUploadResult(); }
From source file:com.amazonaws.services.kinesis.producer.Daemon.java
private void connectToChild() throws IOException { long start = System.nanoTime(); while (true) { try {/*w w w .j av a2 s .c om*/ inChannel = FileChannel.open(Paths.get(inPipe.getAbsolutePath()), StandardOpenOption.READ); outChannel = FileChannel.open(Paths.get(outPipe.getAbsolutePath()), StandardOpenOption.WRITE); outStream = Channels.newOutputStream(outChannel); break; } catch (IOException e) { if (inChannel != null && inChannel.isOpen()) { inChannel.close(); } if (outChannel != null && outChannel.isOpen()) { outChannel.close(); } try { Thread.sleep(100); } catch (InterruptedException e1) { } if (System.nanoTime() - start > 2e9) { throw e; } } } }
From source file:com.lukakama.serviio.watchservice.watcher.WatcherRunnable.java
private boolean isPathFullyAccessible(Path path) { if (!Files.exists(path)) { return false; }// w w w . ja va 2s . c o m if (Files.isDirectory(path)) { DirectoryStream<Path> directoryStream = null; try { directoryStream = Files.newDirectoryStream(path); return true; } catch (IOException e) { log.debug("Unaccessible directory: {}", path, e); return false; } finally { IOUtils.closeQuietly(directoryStream); } } else { FileChannel fileChannel = null; try { fileChannel = FileChannel.open(path, StandardOpenOption.READ); fileChannel.position(Files.size(path)); return true; } catch (IOException e) { log.debug("Unaccessible file: {}", path, e); return false; } finally { IOUtils.closeQuietly(fileChannel); } } }
From source file:com.clustercontrol.agent.log.LogfileMonitor.java
/** * ?/* w w w .j a v a 2 s . c o m*/ */ private boolean openFile() { m_log.info("openFile : filename=" + status.rsFilePath.getName()); closeFile(); FileChannel fc = null; // try { if (checkPrefix()) status.rotate(); fc = FileChannel.open(Paths.get(getFilePath()), StandardOpenOption.READ); long filesize = fc.size(); if (filesize > LogfileMonitorConfig.fileMaxSize) { // ???????? // message.log.agent.1={0}? // message.log.agent.3=?????? // message.log.agent.5={0} byte? String[] args1 = { getFilePath() }; String[] args2 = { String.valueOf(filesize) }; sendMessage(PriorityConstant.TYPE_INFO, MessageConstant.AGENT.getMessage(), MessageConstant.MESSAGE_LOG_FILE_SIZE_EXCEEDED_UPPER_BOUND.getMessage(), MessageConstant.MESSAGE_LOG_FILE.getMessage(args1) + ", " + MessageConstant.MESSAGE_LOG_FILE_SIZE_BYTE.getMessage(args2)); } // ?? // ?open?init=true?? // ??open?init=false??? fc.position(status.position); fileChannel = fc; return true; } catch (FileNotFoundException e) { m_log.info("openFile : " + e.getMessage()); if (m_initFlag) { // ???????? // message.log.agent.1={0}? // message.log.agent.2=??????? String[] args = { getFilePath() }; sendMessage(PriorityConstant.TYPE_INFO, MessageConstant.AGENT.getMessage(), MessageConstant.MESSAGE_LOG_FILE_NOT_FOUND.getMessage(), MessageConstant.MESSAGE_LOG_FILE.getMessage(args)); } return false; } catch (SecurityException e) { m_log.info("openFile : " + e.getMessage()); if (m_initFlag) { // ?? // message.log.agent.1={0}? // message.log.agent.4=???????? String[] args = { getFilePath() }; sendMessage(PriorityConstant.TYPE_WARNING, MessageConstant.AGENT.getMessage(), MessageConstant.MESSAGE_LOG_FAILED_TO_READ_FILE.getMessage(), MessageConstant.MESSAGE_LOG_FILE.getMessage(args) + "\n" + e.getMessage()); } return false; } catch (IOException e) { m_log.info("openFile : " + e.getMessage()); if (m_initFlag) { // ?? // message.log.agent.1={0}? // message.log.agent.4=???????? String[] args = { getFilePath() }; sendMessage(PriorityConstant.TYPE_INFO, MessageConstant.AGENT.getMessage(), MessageConstant.MESSAGE_LOG_FAILED_TO_READ_FILE.getMessage(), MessageConstant.MESSAGE_LOG_FILE.getMessage(args)); } return false; } finally { // ?????????????? if (fc != null && fileChannel == null) { try { fc.close(); } catch (IOException e) { m_log.warn(e.getMessage(), e); } } m_initFlag = false; } }
From source file:com.jivesoftware.os.upena.deployable.UpenaMain.java
public void run(String[] args) throws Exception { HealthFactory.initialize(BindInterfaceToConfiguration::bindDefault, new HealthCheckRegistry() { @Override/*from w w w. ja v a 2 s . com*/ public void register(HealthChecker<?> healthChecker) { } @Override public void unregister(HealthChecker<?> healthChecker) { } }); Properties buildProperties = new Properties(); String upenaVersion = ""; try { buildProperties.load(UpenaMain.class.getClassLoader().getResourceAsStream("build.properties")); upenaVersion = buildProperties.getProperty("my.version", "") + " " + buildProperties.getProperty("my.timestamp", "") + " sha:" + buildProperties.getProperty("git.commit.id", ""); } catch (Exception x) { LOG.warn("Failed to locate build.properties"); } String workingDir = System.getProperty("user.dir"); long start = System.currentTimeMillis(); Exception failed = null; while (start + TimeUnit.SECONDS.toMillis(10) > System.currentTimeMillis()) { try { File lockFile = new File(workingDir, "onlyLetOneRunningAtATime"); lockFile.createNewFile(); FileChannel.open(lockFile.toPath(), StandardOpenOption.WRITE).lock(); failed = null; break; } catch (Exception x) { failed = x; LOG.warn("Failed to acquire lock on onlyLetOneRunningAtATime", x); Thread.sleep(1000); } } if (failed != null) { throw failed; } JDIAPI jvmapi = null; try { jvmapi = new JDIAPI(); } catch (NoClassDefFoundError x) { LOG.warn( "Failed to local tools.jar. Please manually add to classpath. Breakpoint debugger will be disabled."); } String hostname = args[0]; int loopbackPort = Integer.parseInt(System.getProperty("amza.loopback.port", "1174")); int port = Integer.parseInt(System.getProperty("amza.port", "1175")); String multicastGroup = System.getProperty("amza.discovery.group", "225.4.5.6"); int multicastPort = Integer.parseInt(System.getProperty("amza.discovery.port", "1123")); String clusterDiscoveryName = (args.length > 1 ? args[1] : null); String datacenter = System.getProperty("host.datacenter", "unknownDatacenter"); String rack = System.getProperty("host.rack", "unknownRack"); String publicHost = System.getProperty("public.host.name", hostname); UpenaRingHost ringHost = new UpenaRingHost(hostname, port); // TODO include rackId // todo need a better way to create writer id. int writerId = new Random().nextInt(512); TimestampedOrderIdProvider orderIdProvider = new OrderIdProviderImpl( new ConstantWriterIdProvider(writerId)); ObjectMapper mapper = new ObjectMapper(); mapper.configure(SerializationFeature.INDENT_OUTPUT, true); mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); RowsStorageProvider rowsStorageProvider = rowsStorageProvider(orderIdProvider); boolean sslEnable = Boolean.parseBoolean(System.getProperty("ssl.enabled", "true")); String sslKeystorePassword = System.getProperty("ssl.keystore.password", "password"); String sslKeystorePath = System.getProperty("ssl.keystore.path", "./certs/sslKeystore"); String sslKeyStoreAlias = System.getProperty("ssl.keystore.alias", "upenanode").toLowerCase(); boolean sslAutoGenerateSelfSignedCert = Boolean .parseBoolean(System.getProperty("ssl.keystore.autoGenerate", "true")); File sslKeystore = new File(sslKeystorePath); if (sslEnable) { SelfSigningCertGenerator selfSigningCertGenerator = new SelfSigningCertGenerator(); if (sslKeystore.exists()) { if (!selfSigningCertGenerator.validate(sslKeyStoreAlias, sslKeystorePassword, sslKeystore)) { LOG.error("SSL keystore validation failed. keyStoreAlias:{} sslKeystore:{}", sslKeyStoreAlias, sslKeystore); System.exit(1); } } else { sslKeystore.getParentFile().mkdirs(); if (sslAutoGenerateSelfSignedCert) { selfSigningCertGenerator.create(sslKeyStoreAlias, sslKeystorePassword, sslKeystore); } else { LOG.error("Failed to locate mandatory sslKeystore:{}", sslKeystore); System.exit(1); } } } String consumerKey = System.getProperty("upena.consumerKey", clusterDiscoveryName); if (consumerKey == null) { consumerKey = "upena"; LOG.warn("Please provide a stronger consumerKey via -Dupena.consumerKey"); } String finalConsumerKey = consumerKey; String secret = System.getProperty("upena.secret"); if (secret == null) { secret = "secret"; LOG.warn("Please provide a stronger secret via -Dupena.secret"); } String finalSecret = secret; OAuthSigner authSigner = (request) -> { CommonsHttpOAuthConsumer oAuthConsumer = new CommonsHttpOAuthConsumer(finalConsumerKey, finalSecret); oAuthConsumer.setMessageSigner(new HmacSha1MessageSigner()); oAuthConsumer.setTokenWithSecret(finalConsumerKey, finalSecret); return oAuthConsumer.sign(request); }; UpenaSSLConfig upenaSSLConfig = new UpenaSSLConfig(sslEnable, sslAutoGenerateSelfSignedCert, authSigner); UpenaAmzaService upenaAmzaService = null; if (!new File("./state").exists()) { UpdatesSender changeSetSender = new HttpUpdatesSender(sslEnable, sslAutoGenerateSelfSignedCert, authSigner); UpdatesTaker tableTaker = new HttpUpdatesTaker(sslEnable, sslAutoGenerateSelfSignedCert, authSigner); UpenaAmzaServiceConfig upenaAmzaServiceConfig = new UpenaAmzaServiceConfig(); upenaAmzaService = new UpenaAmzaServiceInitializer().initialize(upenaAmzaServiceConfig, orderIdProvider, new com.jivesoftware.os.upena.amza.storage.FstMarshaller( FSTConfiguration.getDefaultConfiguration()), rowsStorageProvider, rowsStorageProvider, rowsStorageProvider, changeSetSender, tableTaker, Optional.<SendFailureListener>absent(), Optional.<UpenaTakeFailureListener>absent(), (changes) -> { }); /*upenaAmzaService.start(ringHost, upenaAmzaServiceConfig.resendReplicasIntervalInMillis, upenaAmzaServiceConfig.applyReplicasIntervalInMillis, upenaAmzaServiceConfig.takeFromNeighborsIntervalInMillis, upenaAmzaServiceConfig.checkIfCompactionIsNeededIntervalInMillis, upenaAmzaServiceConfig.compactTombstoneIfOlderThanNMillis);*/ LOG.info("-----------------------------------------------------------------------"); LOG.info("| OLD Amza Service Online"); LOG.info("-----------------------------------------------------------------------"); } else { LOG.info("-----------------------------------------------------------------------"); LOG.info("| OLD Amza Data is decomissionable"); LOG.info("-----------------------------------------------------------------------"); } BAInterner baInterner = new BAInterner(); AtomicReference<Callable<RingTopology>> topologyProvider = new AtomicReference<>(); // bit of a hack InstanceDescriptor instanceDescriptor = new InstanceDescriptor(datacenter, rack, "", "", "", "", "", "", "", "", 0, "", "", "", 0L, true); ConnectionDescriptorsProvider noAuthConnectionsProvider = (connectionDescriptorsRequest, expectedReleaseGroup) -> { try { RingTopology systemRing = topologyProvider.get().call(); List<ConnectionDescriptor> descriptors = Lists.newArrayList(Iterables.transform(systemRing.entries, input -> new ConnectionDescriptor(instanceDescriptor, sslEnable, false, new HostPort(input.ringHost.getHost(), input.ringHost.getPort()), Collections.emptyMap(), Collections.emptyMap()))); return new ConnectionDescriptorsResponse(200, Collections.emptyList(), "", descriptors, connectionDescriptorsRequest.getRequestUuid()); } catch (Exception e) { throw new RuntimeException(e); } }; TenantsServiceConnectionDescriptorProvider<String> noAuthConnectionPoolProvider = new TenantsServiceConnectionDescriptorProvider<>( Executors.newScheduledThreadPool(1), "", noAuthConnectionsProvider, "", "", 10_000); // TODO config noAuthConnectionPoolProvider.start(); ConnectionDescriptorsProvider connectionsProvider = (connectionDescriptorsRequest, expectedReleaseGroup) -> { try { RingTopology systemRing = topologyProvider.get().call(); List<ConnectionDescriptor> descriptors = Lists.newArrayList(Iterables.transform(systemRing.entries, input -> new ConnectionDescriptor(instanceDescriptor, sslEnable, true, new HostPort(input.ringHost.getHost(), input.ringHost.getPort()), Collections.emptyMap(), Collections.emptyMap()))); return new ConnectionDescriptorsResponse(200, Collections.emptyList(), "", descriptors, connectionDescriptorsRequest.getRequestUuid()); } catch (Exception e) { throw new RuntimeException(e); } }; TenantsServiceConnectionDescriptorProvider<String> connectionPoolProvider = new TenantsServiceConnectionDescriptorProvider<>( Executors.newScheduledThreadPool(1), "", connectionsProvider, "", "", 10_000); // TODO config connectionPoolProvider.start(); HttpDeliveryClientHealthProvider clientHealthProvider = new HttpDeliveryClientHealthProvider("", null, "", 5000, 100); TenantRoutingHttpClientInitializer<String> nonSigningClientInitializer = new TenantRoutingHttpClientInitializer<>( null); TenantAwareHttpClient<String> systemTakeClient = nonSigningClientInitializer .builder(noAuthConnectionPoolProvider, // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).maxConnections(1_000) .socketTimeoutInMillis(60_000).build(); // TODO expose to conf TenantAwareHttpClient<String> stripedTakeClient = nonSigningClientInitializer .builder(noAuthConnectionPoolProvider, // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).maxConnections(1_000) .socketTimeoutInMillis(60_000).build(); // TODO expose to conf TenantRoutingHttpClientInitializer<String> tenantRoutingHttpClientInitializer = new TenantRoutingHttpClientInitializer<>( new OAuthSignerProvider(() -> authSigner)); TenantAwareHttpClient<String> ringClient = tenantRoutingHttpClientInitializer .builder(connectionPoolProvider, // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).maxConnections(1_000) .socketTimeoutInMillis(60_000).build(); // TODO expose to conf AmzaStats amzaStats = new AmzaStats(); AmzaService amzaService = startAmza(workingDir, amzaStats, baInterner, writerId, new RingHost(datacenter, rack, ringHost.getHost(), ringHost.getPort()), new RingMember(ringHost.getHost() + ":" + ringHost.getPort()), authSigner, systemTakeClient, stripedTakeClient, ringClient, topologyProvider, clusterDiscoveryName, multicastGroup, multicastPort); EmbeddedClientProvider embeddedClientProvider = new EmbeddedClientProvider(amzaService); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Amza Service Online"); LOG.info("-----------------------------------------------------------------------"); ObjectMapper storeMapper = new ObjectMapper(); mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); UpenaConfigStore upenaConfigStore = new UpenaConfigStore(orderIdProvider, storeMapper, upenaAmzaService, amzaService, embeddedClientProvider); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Upena Config Store Online"); LOG.info("-----------------------------------------------------------------------"); ExecutorService instanceChangedThreads = Executors.newFixedThreadPool(32); AtomicReference<UbaService> ubaServiceReference = new AtomicReference<>(); UpenaStore upenaStore = new UpenaStore(storeMapper, upenaAmzaService, (instanceChanges) -> { instanceChangedThreads.submit(() -> { UbaService got = ubaServiceReference.get(); if (got != null) { try { got.instanceChanged(instanceChanges); } catch (Exception ex) { ex.printStackTrace(); } } }); }, (changes) -> { }, (change) -> { LOG.info("TODO: tie into conductor. " + change); }, amzaService, embeddedClientProvider); //upenaStore.attachWatchers(); ChaosService chaosService = new ChaosService(upenaStore); SecureRandom random = new SecureRandom(); PasswordStore passwordStore = (key) -> { String password = System.getProperty("sauth.keystore.password"); if (password == null) { File passwordFile = new File(workingDir, "keystore/" + key + ".key"); if (passwordFile.exists()) { password = Files.toString(passwordFile, StandardCharsets.UTF_8); } else { passwordFile.getParentFile().mkdirs(); password = new BigInteger(130, random).toString(32); Files.write(password, passwordFile, StandardCharsets.UTF_8); } } return password; }; SessionStore sessionStore = new SessionStore(TimeUnit.MINUTES.toMillis(60), TimeUnit.MINUTES.toMillis(30)); AtomicReference<UpenaHealth> upenaHealthProvider = new AtomicReference<>(); InstanceHealthly instanceHealthly = (key, version) -> { UpenaHealth upenaHealth = upenaHealthProvider.get(); if (upenaHealth == null) { return false; } ConcurrentMap<RingHost, NodeHealth> ringHostNodeHealth = upenaHealth.buildClusterHealth(); for (NodeHealth nodeHealth : ringHostNodeHealth.values()) { for (NannyHealth nannyHealth : nodeHealth.nannyHealths) { if (nannyHealth.instanceDescriptor.instanceKey.equals(key.getKey())) { return nannyHealth.serviceHealth.fullyOnline ? nannyHealth.serviceHealth.version.equals(version) : false; } } } return false; }; UpenaService upenaService = new UpenaService(passwordStore, sessionStore, upenaStore, chaosService, instanceHealthly); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Upena Service Online"); LOG.info("-----------------------------------------------------------------------"); File defaultPathToRepo = new File(new File(System.getProperty("user.dir"), ".m2"), "repository"); PathToRepo localPathToRepo = new PathToRepo( new File(System.getProperty("pathToRepo", defaultPathToRepo.getAbsolutePath()))); RepositoryProvider repositoryProvider = new RepositoryProvider(localPathToRepo); Host host = new Host(publicHost, datacenter, rack, ringHost.getHost(), ringHost.getPort(), workingDir, null, null); HostKey hostKey = new HostKeyProvider().getNodeKey(upenaStore.hosts, host); String hostInstanceId = System.getProperty("host.instance.id", hostKey.getKey()); host = new Host(publicHost, datacenter, rack, ringHost.getHost(), ringHost.getPort(), workingDir, hostInstanceId, null); UbaLog ubaLog = (what, why, how) -> { try { upenaStore.record("Uba", what, System.currentTimeMillis(), why, ringHost.getHost() + ":" + ringHost.getPort(), how); } catch (Exception x) { x.printStackTrace(); // Hmm lame } }; OktaLog oktaLog = (who, what, why, how) -> { try { upenaStore.record("okta:" + who, what, System.currentTimeMillis(), why, ringHost.getHost() + ":" + ringHost.getPort(), how); } catch (Exception x) { x.printStackTrace(); // Hmm lame } }; OktaCredentialsMatcher.oktaLog = oktaLog; OktaRealm.oktaLog = oktaLog; UpenaClient upenaClient = new UpenaClient() { @Override public InstanceDescriptorsResponse instanceDescriptor( InstanceDescriptorsRequest instanceDescriptorsRequest) throws Exception { return upenaService.instanceDescriptors(instanceDescriptorsRequest); } @Override public void updateKeyPair(String instanceKey, String publicKey) throws Exception { Instance i = upenaStore.instances.get(new InstanceKey(instanceKey)); if (i != null) { LOG.info("Updating publicKey for {}", instanceKey); upenaStore.instances.update(new InstanceKey(instanceKey), new Instance(i.clusterKey, i.hostKey, i.serviceKey, i.releaseGroupKey, i.instanceId, i.enabled, i.locked, publicKey, i.restartTimestampGMTMillis, i.ports)); } } }; final UbaService ubaService = new UbaServiceInitializer().initialize(passwordStore, upenaClient, repositoryProvider, hostKey.getKey(), workingDir, new UbaCoordinate(datacenter, rack, publicHost, host.hostName, "localhost", loopbackPort), null, ubaLog); UpenaHealth upenaHealth = new UpenaHealth(amzaService, upenaSSLConfig, upenaConfigStore, ubaService, new RingHost(datacenter, rack, ringHost.getHost(), ringHost.getPort()), hostKey); upenaHealthProvider.set(upenaHealth); DiscoveredRoutes discoveredRoutes = new DiscoveredRoutes(); ShiroRequestHelper shiroRequestHelper = new ShiroRequestHelper(TimeUnit.DAYS.toMillis(1)); // TODO expose Sys prop? String shiroConfigLocation = System.getProperty("shiro.ini.location", "classpath:shiro.ini"); // classpath:oktashiro.ini UpenaJerseyEndpoints jerseyEndpoints = new UpenaJerseyEndpoints(shiroConfigLocation) .addInjectable(ShiroRequestHelper.class, shiroRequestHelper) .addEndpoint(UpenaClusterRestEndpoints.class).addEndpoint(UpenaHostRestEndpoints.class) .addEndpoint(UpenaServiceRestEndpoints.class).addEndpoint(UpenaReleaseRestEndpoints.class) .addEndpoint(UpenaInstanceRestEndpoints.class).addEndpoint(UpenaTenantRestEndpoints.class) .addInjectable(upenaHealth).addInjectable(upenaService).addInjectable(upenaStore) .addInjectable(upenaConfigStore).addInjectable(ubaService) //.addEndpoint(AmzaReplicationRestEndpoints.class) //.addInjectable(UpenaAmzaInstance.class, upenaAmzaService) .addEndpoint(UpenaEndpoints.class).addEndpoint(UpenaConnectivityEndpoints.class) .addEndpoint(UpenaManagedDeployableEndpoints.class).addEndpoint(UpenaHealthEndpoints.class) .addEndpoint(UpenaRepoEndpoints.class).addInjectable(DiscoveredRoutes.class, discoveredRoutes) .addInjectable(UpenaRingHost.class, ringHost).addInjectable(HostKey.class, hostKey) .addInjectable(UpenaAutoRelease.class, new UpenaAutoRelease(repositoryProvider, upenaStore)) .addInjectable(PathToRepo.class, localPathToRepo); PercentileHealthCheckConfig phcc = bindDefault(PercentileHealthCheckConfig.class); PercentileHealthChecker authFilterHealthCheck = new PercentileHealthChecker(phcc); AuthValidationFilter authValidationFilter = new AuthValidationFilter(authFilterHealthCheck); authValidationFilter.addEvaluator(new NoAuthEvaluator(), "/", "/swagger.json", "/ui/*", // Handled by Shiro "/repo/*" // Cough ); OAuth1Signature verifier = new OAuth1Signature(new OAuthServiceLocatorShim()); OAuthSecretManager oAuthSecretManager = new OAuthSecretManager() { @Override public void clearCache() { } @Override public String getSecret(String id) throws AuthValidationException { return id.equals(finalConsumerKey) ? finalSecret : null; } @Override public void verifyLastSecretRemovalTime() throws Exception { } }; AuthValidator<OAuth1Signature, OAuth1Request> oAuthValidator = new DefaultOAuthValidator( Executors.newScheduledThreadPool(1), Long.MAX_VALUE, oAuthSecretManager, 60_000, false, false); oAuthValidator.start(); authValidationFilter.addEvaluator(new NoAuthEvaluator(), "/repo/*", "/amza/rows/stream/*", "/amza/rows/taken/*", "/amza/pong/*", "/amza/invalidate/*"); authValidationFilter.addEvaluator(new OAuthEvaluator(oAuthValidator, verifier), "/upena/*", "/amza/*"); // TODO something better someday String upenaApiUsername = System.getProperty("upena.api.username", null); String upenaApiPassword = System.getProperty("upena.api.password", null); if (upenaApiUsername != null && upenaApiPassword != null) { authValidationFilter.addEvaluator(containerRequestContext -> { String authCredentials = containerRequestContext.getHeaderString("Authorization"); if (authCredentials == null) { return AuthStatus.not_handled; } final String encodedUserPassword = authCredentials.replaceFirst("Basic" + " ", ""); String usernameAndPassword = null; try { byte[] decodedBytes = Base64.getDecoder().decode(encodedUserPassword); usernameAndPassword = new String(decodedBytes, "UTF-8"); } catch (IOException e) { return AuthStatus.denied; } final StringTokenizer tokenizer = new StringTokenizer(usernameAndPassword, ":"); final String username = tokenizer.nextToken(); final String password = tokenizer.nextToken(); boolean authenticationStatus = upenaApiUsername.equals(username) && upenaApiPassword.equals(password); return authenticationStatus ? AuthStatus.authorized : AuthStatus.denied; }, "/api/*"); } jerseyEndpoints.addContainerRequestFilter(authValidationFilter); String region = System.getProperty("aws.region", null); String roleArn = System.getProperty("aws.roleArn", null); AWSClientFactory awsClientFactory = new AWSClientFactory(region, roleArn); String accountName = System.getProperty("account.name", clusterDiscoveryName == null ? "" : clusterDiscoveryName); String humanReadableUpenaClusterName = datacenter + " - " + accountName; injectUI(upenaVersion, awsClientFactory, storeMapper, mapper, jvmapi, amzaService, localPathToRepo, repositoryProvider, hostKey, ringHost, upenaSSLConfig, port, sessionStore, ubaService, upenaHealth, upenaStore, upenaConfigStore, jerseyEndpoints, humanReadableUpenaClusterName, discoveredRoutes); injectAmza(baInterner, amzaStats, jerseyEndpoints, amzaService, ringClient); InitializeRestfulServer initializeRestfulServer = new InitializeRestfulServer(false, port, "UpenaNode", sslEnable, sslKeyStoreAlias, sslKeystorePassword, sslKeystorePath, 128, 10_000); buildSwagger(); initializeRestfulServer.addClasspathResource("/resources"); initializeRestfulServer.addContextHandler("/", jerseyEndpoints); RestfulServer restfulServer = initializeRestfulServer.build(); restfulServer.start(); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Jetty Service Online"); LOG.info("-----------------------------------------------------------------------"); UpenaJerseyEndpoints loopbackJerseyEndpoints = new UpenaJerseyEndpoints(null) .addEndpoint(UpenaLoopbackEndpoints.class).addEndpoint(UpenaConfigRestEndpoints.class) .addInjectable(SessionStore.class, sessionStore) .addInjectable(DiscoveredRoutes.class, discoveredRoutes).addInjectable(upenaConfigStore) .addInjectable(upenaStore).addInjectable(upenaHealth) .addInjectable(UpenaService.class, upenaService); InitializeRestfulServer initializeLoopbackRestfulServer = new InitializeRestfulServer( Boolean.parseBoolean(System.getProperty("amza.loopback.strict", "true")), loopbackPort, "UpenaNode", false, sslKeyStoreAlias, sslKeystorePassword, sslKeystorePath, 128, 10_000); initializeLoopbackRestfulServer.addClasspathResource("/resources"); initializeLoopbackRestfulServer.addContextHandler("/", loopbackJerseyEndpoints); RestfulServer loopbackRestfulServer = initializeLoopbackRestfulServer.build(); loopbackRestfulServer.start(); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Jetty Service Online"); LOG.info("-----------------------------------------------------------------------"); if (ubaService != null) { Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(() -> { try { ubaService.nanny(); } catch (Exception ex) { LOG.error("Nanny failure", ex); } }, 15, 15, TimeUnit.SECONDS); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Uba Service Online"); LOG.info("-----------------------------------------------------------------------"); } ubaServiceReference.set(ubaService); /*String peers = System.getProperty("manual.peers"); if (peers != null) { String[] hostPortTuples = peers.split(","); for (String hostPortTuple : hostPortTuples) { String hostPort = hostPortTuple.trim(); if (hostPort.length() > 0 && hostPort.contains(":")) { String[] host_port = hostPort.split(":"); try { UpenaRingHost anotherRingHost = new UpenaRingHost(host_port[0].trim(), Integer.parseInt(host_port[1].trim())); List<UpenaRingHost> ring = upenaAmzaService.getRing("master"); if (!ring.contains(anotherRingHost)) { LOG.info("Adding host to the cluster: " + anotherRingHost); upenaAmzaService.addRingHost("master", anotherRingHost); } } catch (Exception x) { LOG.warn("Malformed hostPortTuple {}", hostPort); } } else { LOG.warn("Malformed hostPortTuple {}", hostPort); } } }*/ String vpc = System.getProperty("aws.vpc", null); UpenaAWSLoadBalancerNanny upenaAWSLoadBalancerNanny = new UpenaAWSLoadBalancerNanny(vpc, upenaStore, hostKey, awsClientFactory); Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(() -> { try { upenaAWSLoadBalancerNanny.ensureSelf(); } catch (Exception x) { LOG.warn("Failures while nannying load loadbalancer.", x); } }, 1, 1, TimeUnit.MINUTES); // TODO better LOG.info("-----------------------------------------------------------------------"); LOG.info("| Waiting for amza to be ready...."); LOG.info("-----------------------------------------------------------------------"); while (!amzaService.isReady()) { Thread.sleep(1000); } LOG.info("-----------------------------------------------------------------------"); LOG.info("| Begin Migration"); LOG.info("-----------------------------------------------------------------------"); upenaStore.init(orderIdProvider, Integer.parseInt(System.getProperty("min.service.port", "10000")), Integer.parseInt(System.getProperty("max.service.port", String.valueOf(Short.MAX_VALUE))), false); upenaConfigStore.init(); LOG.info("-----------------------------------------------------------------------"); LOG.info("| End Migration"); LOG.info("-----------------------------------------------------------------------"); addManualPeers(amzaService); Host gotHost = upenaStore.hosts.get(hostKey); if (gotHost == null || !gotHost.equals(host)) { upenaStore.hosts.update(hostKey, host); } }
From source file:com.spectralogic.ds3client.integration.Smoke_Test.java
@Test public void verifySendCrc32cChecksum() throws IOException, XmlProcessingException, URISyntaxException { final String bucketName = "crc_32_bucket"; final String dataPolicyName = "crc_32_dp"; final String storageDomainName = "crc_32_sd"; final String poolPartitionName = "crc_32_pp"; UUID storageDomainMemberId = null; UUID dataPersistenceRuleId = null; try {//from w w w. j av a 2 s .c o m //Create data policy final PutDataPolicySpectraS3Response dataPolicyResponse = createDataPolicyWithVersioningAndCrcRequired( dataPolicyName, VersioningLevel.NONE, ChecksumType.Type.CRC_32C, client); //Create storage domain final PutStorageDomainSpectraS3Response storageDomainResponse = createStorageDomain(storageDomainName, client); //Create pool partition final PutPoolPartitionSpectraS3Response poolPartitionResponse = createPoolPartition(poolPartitionName, PoolType.ONLINE, client); //Create storage domain member linking pool partition to storage domain final PutPoolStorageDomainMemberSpectraS3Response memberResponse = createPoolStorageDomainMember( storageDomainResponse.getStorageDomainResult().getId(), poolPartitionResponse.getPoolPartitionResult().getId(), client); storageDomainMemberId = memberResponse.getStorageDomainMemberResult().getId(); //create data persistence rule final PutDataPersistenceRuleSpectraS3Response dataPersistenceResponse = createDataPersistenceRule( dataPolicyResponse.getDataPolicyResult().getId(), storageDomainResponse.getStorageDomainResult().getId(), client); dataPersistenceRuleId = dataPersistenceResponse.getDataPersistenceRuleResult().getDataPolicyId(); //Create bucket with data policy client.putBucketSpectraS3(new PutBucketSpectraS3Request(bucketName) .withDataPolicyId(dataPolicyResponse.getDataPolicyResult().getId().toString())); //Verify send CRC 32c checksum final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); helpers.ensureBucketExists(bucketName); final List<Ds3Object> objs = Lists.newArrayList(new Ds3Object("beowulf.txt", 294059)); final MasterObjectList mol = client .putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucketName, objs)).getResult(); final FileChannel channel = FileChannel.open(ResourceUtils.loadFileResource("books/beowulf.txt"), StandardOpenOption.READ); final PutObjectResponse response = client.putObject( new PutObjectRequest(bucketName, "beowulf.txt", channel, mol.getJobId().toString(), 0, 294059) .withChecksum(ChecksumType.compute(), ChecksumType.Type.CRC_32C)); assertThat(response.getChecksumType(), is(ChecksumType.Type.CRC_32C)); assertThat(response.getChecksum(), is("+ZBZbQ==")); } finally { deleteAllContents(client, bucketName); deleteDataPersistenceRule(dataPersistenceRuleId, client); deleteDataPolicy(dataPolicyName, client); deleteStorageDomainMember(storageDomainMemberId, client); deleteStorageDomain(storageDomainName, client); deletePoolPartition(poolPartitionName, client); } }
From source file:edu.harvard.iq.dataverse.dataaccess.TabularSubsetGenerator.java
public Object[] subsetObjectVector(File tabfile, int column, int varcount, int casecount, int columntype, boolean compatmode) throws IOException { Object[] retVector = null;/*from ww w.j a v a 2 s. c o m*/ boolean isString = false; boolean isDouble = false; boolean isLong = false; boolean isFloat = false; //Locale loc = new Locale("en", "US"); if (columntype == COLUMN_TYPE_STRING) { isString = true; retVector = new String[casecount]; } else if (columntype == COLUMN_TYPE_DOUBLE) { isDouble = true; retVector = new Double[casecount]; } else if (columntype == COLUMN_TYPE_LONG) { isLong = true; retVector = new Long[casecount]; } else if (columntype == COLUMN_TYPE_FLOAT) { isFloat = true; retVector = new Float[casecount]; } else { throw new IOException("Unsupported column type: " + columntype); } File rotatedImageFile = getRotatedImage(tabfile, varcount, casecount); long[] columnEndOffsets = extractColumnOffsets(rotatedImageFile, varcount, casecount); long columnOffset = 0; long columnLength = 0; if (column > 0) { columnOffset = columnEndOffsets[column - 1]; columnLength = columnEndOffsets[column] - columnEndOffsets[column - 1]; } else { columnOffset = varcount * 8; columnLength = columnEndOffsets[0] - varcount * 8; } FileChannel fc = (FileChannel.open(Paths.get(rotatedImageFile.getAbsolutePath()), StandardOpenOption.READ)); fc.position(columnOffset); int MAX_COLUMN_BUFFER = 8192; ByteBuffer in = ByteBuffer.allocate(MAX_COLUMN_BUFFER); if (columnLength < MAX_COLUMN_BUFFER) { in.limit((int) (columnLength)); } long bytesRead = 0; long bytesReadTotal = 0; int caseindex = 0; int byteoffset = 0; byte[] leftover = null; while (bytesReadTotal < columnLength) { bytesRead = fc.read(in); byte[] columnBytes = in.array(); int bytecount = 0; while (bytecount < bytesRead) { if (columnBytes[bytecount] == '\n') { /* String token = new String(columnBytes, byteoffset, bytecount-byteoffset, "UTF8"); if (leftover != null) { String leftoverString = new String (leftover, "UTF8"); token = leftoverString + token; leftover = null; } */ /* * Note that the way I was doing it at first - above - * was not quite the correct way - because I was creating UTF8 * strings from the leftover bytes, and the bytes in the * current buffer *separately*; which means, if a multi-byte * UTF8 character got split in the middle between one buffer * and the next, both chunks of it would become junk * characters, on each side! * The correct way of doing it, of course, is to create a * merged byte buffer, and then turn it into a UTF8 string. * -- L.A. 4.0 */ String token = null; if (leftover == null) { token = new String(columnBytes, byteoffset, bytecount - byteoffset, "UTF8"); } else { byte[] merged = new byte[leftover.length + bytecount - byteoffset]; System.arraycopy(leftover, 0, merged, 0, leftover.length); System.arraycopy(columnBytes, byteoffset, merged, leftover.length, bytecount - byteoffset); token = new String(merged, "UTF8"); leftover = null; merged = null; } if (isString) { if ("".equals(token)) { // An empty string is a string missing value! // An empty string in quotes is an empty string! retVector[caseindex] = null; } else { // Strip the outer quotes: token = token.replaceFirst("^\\\"", ""); token = token.replaceFirst("\\\"$", ""); // We need to restore the special characters that // are stored in tab files escaped - quotes, new lines // and tabs. Before we do that however, we need to // take care of any escaped backslashes stored in // the tab file. I.e., "foo\t" should be transformed // to "foo<TAB>"; but "foo\\t" should be transformed // to "foo\t". This way new lines and tabs that were // already escaped in the original data are not // going to be transformed to unescaped tab and // new line characters! String[] splitTokens = token.split(Matcher.quoteReplacement("\\\\"), -2); // (note that it's important to use the 2-argument version // of String.split(), and set the limit argument to a // negative value; otherwise any trailing backslashes // are lost.) for (int i = 0; i < splitTokens.length; i++) { splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\\""), "\""); splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\t"), "\t"); splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\n"), "\n"); splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\r"), "\r"); } // TODO: // Make (some of?) the above optional; for ex., we // do need to restore the newlines when calculating UNFs; // But if we are subsetting these vectors in order to // create a new tab-delimited file, they will // actually break things! -- L.A. Jul. 28 2014 token = StringUtils.join(splitTokens, '\\'); // "compatibility mode" - a hack, to be able to produce // unfs identical to those produced by the "early" // unf5 jar; will be removed in production 4.0. // -- L.A. (TODO: ...) if (compatmode && !"".equals(token)) { if (token.length() > 128) { if ("".equals(token.trim())) { // don't ask... token = token.substring(0, 129); } else { token = token.substring(0, 128); //token = String.format(loc, "%.128s", token); token = token.trim(); //dbgLog.info("formatted and trimmed: "+token); } } else { if ("".equals(token.trim())) { // again, don't ask; // - this replicates some bugginness // that happens inside unf5; token = "null"; } else { token = token.trim(); } } } retVector[caseindex] = token; } } else if (isDouble) { try { // TODO: verify that NaN and +-Inf are // handled correctly here! -- L.A. // Verified: new Double("nan") works correctly, // resulting in Double.NaN; // Double("[+-]Inf") doesn't work however; // (the constructor appears to be expecting it // to be spelled as "Infinity", "-Infinity", etc. if ("inf".equalsIgnoreCase(token) || "+inf".equalsIgnoreCase(token)) { retVector[caseindex] = java.lang.Double.POSITIVE_INFINITY; } else if ("-inf".equalsIgnoreCase(token)) { retVector[caseindex] = java.lang.Double.NEGATIVE_INFINITY; } else if (token == null || token.equals("")) { // missing value: retVector[caseindex] = null; } else { retVector[caseindex] = new Double(token); } } catch (NumberFormatException ex) { dbgLog.warning("NumberFormatException thrown for " + token + " as Double"); retVector[caseindex] = null; // missing value // TODO: ? } } else if (isLong) { try { retVector[caseindex] = new Long(token); } catch (NumberFormatException ex) { retVector[caseindex] = null; // assume missing value } } else if (isFloat) { try { if ("inf".equalsIgnoreCase(token) || "+inf".equalsIgnoreCase(token)) { retVector[caseindex] = java.lang.Float.POSITIVE_INFINITY; } else if ("-inf".equalsIgnoreCase(token)) { retVector[caseindex] = java.lang.Float.NEGATIVE_INFINITY; } else if (token == null || token.equals("")) { // missing value: retVector[caseindex] = null; } else { retVector[caseindex] = new Float(token); } } catch (NumberFormatException ex) { dbgLog.warning("NumberFormatException thrown for " + token + " as Float"); retVector[caseindex] = null; // assume missing value (TODO: ?) } } caseindex++; if (bytecount == bytesRead - 1) { byteoffset = 0; } else { byteoffset = bytecount + 1; } } else { if (bytecount == bytesRead - 1) { // We've reached the end of the buffer; // This means we'll save whatever unused bytes left in // it - i.e., the bytes between the last new line // encountered and the end - in the leftover buffer. // *EXCEPT*, there may be a case of a very long String // that is actually longer than MAX_COLUMN_BUFFER, in // which case it is possible that we've read through // an entire buffer of bytes without finding any // new lines... in this case we may need to add this // entire byte buffer to an already existing leftover // buffer! if (leftover == null) { leftover = new byte[(int) bytesRead - byteoffset]; System.arraycopy(columnBytes, byteoffset, leftover, 0, (int) bytesRead - byteoffset); } else { if (byteoffset != 0) { throw new IOException( "Reached the end of the byte buffer, with some leftover left from the last read; yet the offset is not zero!"); } byte[] merged = new byte[leftover.length + (int) bytesRead]; System.arraycopy(leftover, 0, merged, 0, leftover.length); System.arraycopy(columnBytes, byteoffset, merged, leftover.length, (int) bytesRead); //leftover = null; leftover = merged; merged = null; } byteoffset = 0; } } bytecount++; } bytesReadTotal += bytesRead; in.clear(); if (columnLength - bytesReadTotal < MAX_COLUMN_BUFFER) { in.limit((int) (columnLength - bytesReadTotal)); } } fc.close(); if (caseindex != casecount) { throw new IOException("Faile to read " + casecount + " tokens for column " + column); //System.out.println("read "+caseindex+" tokens instead of expected "+casecount+"."); } return retVector; }