Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.config.store.hdfs; import java.io.BufferedWriter; import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; import java.util.List; import java.util.Properties; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; import com.typesafe.config.Config; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import gobblin.config.common.impl.SingleLinkedListConfigKeyPath; import gobblin.config.store.api.ConfigKeyPath; import gobblin.config.store.api.ConfigStoreCreationException; import gobblin.config.store.deploy.ClasspathConfigSource; import gobblin.config.store.deploy.FsDeploymentConfig; import gobblin.util.PathUtils; /** * Unit tests for {@link SimpleHDFSConfigStore} and {@link SimpleHDFSConfigStoreFactory}. */ @Test(groups = "gobblin.config.store.hdfs") public class SimpleHdfsConfigStoreTest { private static final String CONFIG_DIR_NAME = "configDir"; private static final String VERSION = "v1.0"; private static final Path CONFIG_DIR_PATH = PathUtils.combinePaths(CONFIG_DIR_NAME, SimpleHDFSConfigStore.CONFIG_STORE_NAME, VERSION); /**Set by {@link TestEnvironment#setup()}**/ public static final String TAG_NAME_SYS_PROP_KEY = "sysProp.tagName1"; public static final String TAG_NAME_SYS_PROP_VALUE = "tag1"; private FileSystem fs; private SimpleHDFSConfigStore simpleHDFSConfigStore; @BeforeClass public void setUp() throws URISyntaxException, ConfigStoreCreationException, IOException { this.fs = FileSystem.getLocal(new Configuration()); this.fs.mkdirs(CONFIG_DIR_PATH); SimpleLocalHDFSConfigStoreFactory simpleHDFSConfigStoreConfigFactory = new SimpleLocalHDFSConfigStoreFactory(); URI storeURI = getStoreURI(System.getProperty("user.dir") + File.separator + CONFIG_DIR_NAME); this.simpleHDFSConfigStore = simpleHDFSConfigStoreConfigFactory.createConfigStore(storeURI); this.simpleHDFSConfigStore .deploy(new FsDeploymentConfig(new ClasspathConfigSource(new Properties()), VERSION)); } @Test public void testGetCurrentVersion() throws IOException { Assert.assertEquals(this.simpleHDFSConfigStore.getCurrentVersion(), VERSION); String newVersion = "v1.1"; this.simpleHDFSConfigStore .deploy(new FsDeploymentConfig(new ClasspathConfigSource(new Properties()), newVersion)); Assert.assertEquals(this.simpleHDFSConfigStore.getCurrentVersion(), newVersion); } @Test public void getStoreURI() { URI storeURI = this.simpleHDFSConfigStore.getStoreURI(); Assert.assertEquals(storeURI.getScheme(), SimpleHDFSConfigStoreFactory.SIMPLE_HDFS_SCHEME_PREFIX + "file"); Assert.assertNull(storeURI.getAuthority()); Assert.assertEquals(storeURI.getPath(), System.getProperty("user.dir") + File.separator + CONFIG_DIR_NAME); } @Test public void testGetChildren() throws IOException, URISyntaxException, ConfigStoreCreationException { String datasetName = "dataset-test-get-children"; String childDatasetName = "childDataset"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(new Path(datasetPath, childDatasetName)); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); Collection<ConfigKeyPath> children = this.simpleHDFSConfigStore.getChildren(datasetConfigKey, VERSION); Assert.assertEquals(children.size(), 1); Assert.assertEquals(children.iterator().next().getOwnPathName(), childDatasetName); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } } @Test public void testGetOwnImports() throws IOException, URISyntaxException, ConfigStoreCreationException { String datasetName = "dataset-test-get-own-imports"; String tagKey1 = "/path/to/tag1"; String tagKey2 = "/path/to/tag2"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(datasetPath); BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8)); writer.write(tagKey1); writer.newLine(); writer.write(tagKey2); writer.close(); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); List<ConfigKeyPath> imports = this.simpleHDFSConfigStore.getOwnImports(datasetConfigKey, VERSION); Assert.assertEquals(imports.size(), 2); Assert.assertEquals(imports.get(0).getAbsolutePathString(), tagKey2); Assert.assertEquals(imports.get(1).getAbsolutePathString(), tagKey1); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } } @Test public void testGetOwnImportsWithResolution() throws IOException, URISyntaxException, ConfigStoreCreationException { String datasetName = "dataset-test-get-own-imports-resolution"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(datasetPath); BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8)); writer.write("/path/to/${?" + TAG_NAME_SYS_PROP_KEY + "}"); writer.close(); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); List<ConfigKeyPath> imports = this.simpleHDFSConfigStore.getOwnImports(datasetConfigKey, VERSION); Assert.assertEquals(imports.size(), 1); Assert.assertEquals(imports.get(0).getAbsolutePathString(), "/path/to/" + TAG_NAME_SYS_PROP_VALUE); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } } @Test public void testGetOwnConfig() throws ConfigStoreCreationException, URISyntaxException, IOException { String datasetName = "dataset-test-get-own-config"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(datasetPath); this.fs.create(new Path(datasetPath, "main.conf")).close(); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); Config config = this.simpleHDFSConfigStore.getOwnConfig(datasetConfigKey, VERSION); Assert.assertTrue(config.isEmpty()); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } } @Test(dependsOnMethods = { "testGetCurrentVersion" }) public void testDeploy() throws Exception { Properties props = new Properties(); props.setProperty(ClasspathConfigSource.CONFIG_STORE_CLASSPATH_RESOURCE_NAME_KEY, "_testDeploy"); this.simpleHDFSConfigStore.deploy(new FsDeploymentConfig(new ClasspathConfigSource(props), "2.0")); Path versionPath = PathUtils.combinePaths(CONFIG_DIR_NAME, SimpleHDFSConfigStore.CONFIG_STORE_NAME, "2.0"); Assert.assertTrue(fs.exists(new Path(versionPath, "dir1"))); Assert.assertTrue(fs.exists(new Path(versionPath, "dir1/f1.conf"))); } @Test public void testResolveImports() throws Exception { List<String> unresolved = ImmutableList.of("/path/to/tag0", "/path/to/${?" + TAG_NAME_SYS_PROP_KEY + "}", "${?" + TAG_NAME_SYS_PROP_KEY + "}/${?" + TAG_NAME_SYS_PROP_KEY + "}"); List<String> resolved = SimpleHDFSConfigStore.resolveIncludesList(unresolved); List<String> expected = ImmutableList.of("/path/to/tag0", "/path/to/tag1", "tag1/tag1"); Assert.assertEquals(resolved, expected); } @AfterClass public void tearDown() throws IOException { if (this.fs.exists(new Path(CONFIG_DIR_NAME))) { this.fs.delete(new Path(CONFIG_DIR_NAME), true); } } private URI getStoreURI(String configDir) throws URISyntaxException { return new URI(SimpleHDFSConfigStoreFactory.SIMPLE_HDFS_SCHEME_PREFIX + "file", "localhost:8080", configDir, "", ""); } }