Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.zeppelin.rest; import java.io.File; import java.io.IOException; import java.lang.ref.WeakReference; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethodBase; import org.apache.commons.httpclient.methods.*; import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.server.ZeppelinServer; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonElement; import com.google.gson.JsonParseException; import com.google.gson.JsonParser; public abstract class AbstractTestRestApi { protected static final Logger LOG = LoggerFactory.getLogger(AbstractTestRestApi.class); static final String restApiUrl = "/api"; static final String url = getUrlToTest(); protected static final boolean wasRunning = checkIfServerIsRuning(); static boolean pySpark = false; private String getUrl(String path) { String url; if (System.getProperty("url") != null) { url = System.getProperty("url"); } else { url = "http://localhost:8080"; } url += restApiUrl; if (path != null) url += path; return url; } protected static String getUrlToTest() { String url = "http://localhost:8080" + restApiUrl; if (System.getProperty("url") != null) { url = System.getProperty("url"); } return url; } static ExecutorService executor; protected static final Runnable server = new Runnable() { @Override public void run() { try { ZeppelinServer.main(new String[] { "" }); } catch (Exception e) { e.printStackTrace(); throw new RuntimeException(e); } } }; protected static void startUp() throws Exception { if (!wasRunning) { LOG.info("Staring test Zeppelin up..."); executor = Executors.newSingleThreadExecutor(); executor.submit(server); long s = System.currentTimeMillis(); boolean started = false; while (System.currentTimeMillis() - s < 1000 * 60 * 3) { // 3 minutes Thread.sleep(2000); started = checkIfServerIsRuning(); if (started == true) { break; } } if (started == false) { throw new RuntimeException("Can not start Zeppelin server"); } LOG.info("Test Zeppelin stared."); // ci environment runs spark cluster for testing // so configure zeppelin use spark cluster if ("true".equals(System.getenv("CI"))) { // assume first one is spark InterpreterSetting sparkIntpSetting = null; for (InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) { if (intpSetting.getGroup().equals("spark")) { sparkIntpSetting = intpSetting; } } // set spark master sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); // set spark home for pyspark sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome()); pySpark = true; ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.id()); } else { // assume first one is spark InterpreterSetting sparkIntpSetting = null; for (InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) { if (intpSetting.getGroup().equals("spark")) { sparkIntpSetting = intpSetting; } } String sparkHome = getSparkHome(); if (sparkHome != null) { // set spark home for pyspark sparkIntpSetting.getProperties().setProperty("spark.home", sparkHome); pySpark = true; } ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.id()); } } } private static String getHostname() { try { return InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { e.printStackTrace(); return "localhost"; } } private static String getSparkHome() { String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir"))); System.out.println("SPARK HOME detected " + sparkHome); return sparkHome; } boolean isPyspark() { return pySpark; } private static String getSparkHomeRecursively(File dir) { if (dir == null) return null; File files[] = dir.listFiles(); if (files == null) return null; File homeDetected = null; for (File f : files) { if (isActiveSparkHome(f)) { homeDetected = f; break; } } if (homeDetected != null) { return homeDetected.getAbsolutePath(); } else { return getSparkHomeRecursively(dir.getParentFile()); } } private static boolean isActiveSparkHome(File dir) { if (dir.getName().matches("spark-[0-9\\.]+-bin-hadoop[0-9\\.]+")) { File pidDir = new File(dir, "run"); if (pidDir.isDirectory() && pidDir.listFiles().length > 0) { return true; } } return false; } protected static void shutDown() throws Exception { if (!wasRunning) { // restart interpreter to stop all interpreter processes List<String> settingList = ZeppelinServer.notebook.getInterpreterFactory() .getDefaultInterpreterSettingList(); for (String setting : settingList) { ZeppelinServer.notebook.getInterpreterFactory().restart(setting); } LOG.info("Terminating test Zeppelin..."); ZeppelinServer.jettyServer.stop(); executor.shutdown(); long s = System.currentTimeMillis(); boolean started = true; while (System.currentTimeMillis() - s < 1000 * 60 * 3) { // 3 minutes Thread.sleep(2000); started = checkIfServerIsRuning(); if (started == false) { break; } } if (started == true) { throw new RuntimeException("Can not stop Zeppelin server"); } LOG.info("Test Zeppelin terminated."); } } protected static boolean checkIfServerIsRuning() { GetMethod request = null; boolean isRunning = true; try { request = httpGet("/"); isRunning = request.getStatusCode() == 200; } catch (IOException e) { isRunning = false; } finally { if (request != null) { request.releaseConnection(); } } return isRunning; } protected static GetMethod httpGet(String path) throws IOException { LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); GetMethod getMethod = new GetMethod(url + path); getMethod.addRequestHeader("Origin", url); httpClient.executeMethod(getMethod); LOG.info("{} - {}", getMethod.getStatusCode(), getMethod.getStatusText()); return getMethod; } protected static DeleteMethod httpDelete(String path) throws IOException { LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); DeleteMethod deleteMethod = new DeleteMethod(url + path); deleteMethod.addRequestHeader("Origin", url); httpClient.executeMethod(deleteMethod); LOG.info("{} - {}", deleteMethod.getStatusCode(), deleteMethod.getStatusText()); return deleteMethod; } protected static PostMethod httpPost(String path, String body) throws IOException { LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); PostMethod postMethod = new PostMethod(url + path); postMethod.addRequestHeader("Origin", url); RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8")); postMethod.setRequestEntity(entity); httpClient.executeMethod(postMethod); LOG.info("{} - {}", postMethod.getStatusCode(), postMethod.getStatusText()); return postMethod; } protected static PutMethod httpPut(String path, String body) throws IOException { LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); PutMethod putMethod = new PutMethod(url + path); putMethod.addRequestHeader("Origin", url); RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8")); putMethod.setRequestEntity(entity); httpClient.executeMethod(putMethod); LOG.info("{} - {}", putMethod.getStatusCode(), putMethod.getStatusText()); return putMethod; } protected Matcher<HttpMethodBase> responsesWith(final int expectedStatusCode) { return new TypeSafeMatcher<HttpMethodBase>() { WeakReference<HttpMethodBase> method; @Override public boolean matchesSafely(HttpMethodBase httpMethodBase) { method = (method == null) ? new WeakReference<HttpMethodBase>(httpMethodBase) : method; return httpMethodBase.getStatusCode() == expectedStatusCode; } @Override public void describeTo(Description description) { description.appendText("HTTP response ").appendValue(expectedStatusCode).appendText(" from ") .appendText(method.get().getPath()); } @Override protected void describeMismatchSafely(HttpMethodBase item, Description description) { description.appendText("got ").appendValue(item.getStatusCode()).appendText(" ") .appendText(item.getStatusText()); } }; } protected TypeSafeMatcher<String> isJSON() { return new TypeSafeMatcher<String>() { @Override public boolean matchesSafely(String body) { String b = body.trim(); return (b.startsWith("{") && b.endsWith("}")) || (b.startsWith("[") && b.endsWith("]")); } @Override public void describeTo(Description description) { description.appendText("response in JSON format "); } @Override protected void describeMismatchSafely(String item, Description description) { description.appendText("got ").appendText(item); } }; } protected TypeSafeMatcher<String> isValidJSON() { return new TypeSafeMatcher<String>() { @Override public boolean matchesSafely(String body) { boolean isValid = true; try { new JsonParser().parse(body); } catch (JsonParseException e) { isValid = false; } return isValid; } @Override public void describeTo(Description description) { description.appendText("response in JSON format "); } @Override protected void describeMismatchSafely(String item, Description description) { description.appendText("got ").appendText(item); } }; } //Create new Setting and return Setting ID protected String createTempSetting(String tempName) throws IOException { InterpreterGroup interpreterGroup = ZeppelinServer.notebook.getInterpreterFactory().add(tempName, "newGroup", new InterpreterOption(false), new Properties()); return interpreterGroup.getId(); } protected TypeSafeMatcher<? super JsonElement> hasRootElementNamed(final String memberName) { return new TypeSafeMatcher<JsonElement>() { @Override protected boolean matchesSafely(JsonElement item) { return item.isJsonObject() && item.getAsJsonObject().has(memberName); } @Override public void describeTo(Description description) { description.appendText("response in JSON format with \"").appendText(memberName) .appendText("\" beeing a root element "); } @Override protected void describeMismatchSafely(JsonElement root, Description description) { description.appendText("got ").appendText(root.toString()); } }; } /** Status code matcher */ protected Matcher<? super HttpMethodBase> isForbiden() { return responsesWith(403); } protected Matcher<? super HttpMethodBase> isAllowed() { return responsesWith(200); } protected Matcher<? super HttpMethodBase> isCreated() { return responsesWith(201); } protected Matcher<? super HttpMethodBase> isNotAllowed() { return responsesWith(405); } }