Java tutorial
// -*- mode: java; c-basic-offset: 2; -*- // Copyright 2009-2011 Google, All Rights reserved // Copyright 2011-2012 MIT, All rights reserved // Released under the MIT License https://raw.github.com/mit-cml/app-inventor/master/mitlicense.txt package com.google.appinventor.buildserver; import com.google.appinventor.common.utils.StringUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.io.Files; import com.google.common.io.InputSupplier; import com.google.common.io.Resources; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.Enumeration; import java.util.List; import java.util.Set; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.apache.commons.io.FileUtils; /** * Provides support for building Young Android projects. * * @author markf@google.com (Mark Friedman) */ public final class ProjectBuilder { private File outputApk; private File outputKeystore; private boolean saveKeystore; // Logging support private static final Logger LOG = Logger.getLogger(ProjectBuilder.class.getName()); private static final int MAX_COMPILER_MESSAGE_LENGTH = 160; // Project folder prefixes // TODO(user): These constants are (or should be) also defined in // appengine/src/com/google/appinventor/server/project/youngandroid/YoungAndroidProjectService // They should probably be in some place shared with the server private static final String PROJECT_DIRECTORY = "youngandroidproject"; private static final String PROJECT_PROPERTIES_FILE_NAME = PROJECT_DIRECTORY + "/" + "project.properties"; private static final String KEYSTORE_FILE_NAME = YoungAndroidConstants.PROJECT_KEYSTORE_LOCATION; private static final String FORM_PROPERTIES_EXTENSION = YoungAndroidConstants.FORM_PROPERTIES_EXTENSION; private static final String YAIL_EXTENSION = YoungAndroidConstants.YAIL_EXTENSION; private static final String CODEBLOCKS_SOURCE_EXTENSION = YoungAndroidConstants.CODEBLOCKS_SOURCE_EXTENSION; private static final String ALL_COMPONENT_TYPES = Compiler.RUNTIME_FILES_DIR + "simple_components.txt"; public File getOutputApk() { return outputApk; } public File getOutputKeystore() { return outputKeystore; } /** * Creates a new directory beneath the system's temporary directory (as * defined by the {@code java.io.tmpdir} system property), and returns its * name. The name of the directory will contain the current time (in millis), * and a random number. * * <p>This method assumes that the temporary volume is writable, has free * inodes and free blocks, and that it will not be called thousands of times * per second. * * @return the newly-created directory * @throws IllegalStateException if the directory could not be created */ private static File createNewTempDir() { File baseDir = new File(System.getProperty("java.io.tmpdir")); String baseNamePrefix = System.currentTimeMillis() + "_" + Math.random() + "-"; final int TEMP_DIR_ATTEMPTS = 10000; for (int counter = 0; counter < TEMP_DIR_ATTEMPTS; counter++) { File tempDir = new File(baseDir, baseNamePrefix + counter); if (tempDir.exists()) { continue; } if (tempDir.mkdir()) { return tempDir; } } throw new IllegalStateException("Failed to create directory within " + TEMP_DIR_ATTEMPTS + " attempts (tried " + baseNamePrefix + "0 to " + baseNamePrefix + (TEMP_DIR_ATTEMPTS - 1) + ')'); } Result build(String userName, ZipFile inputZip, File outputDir, boolean isForRepl, boolean isForWireless, int childProcessRam, String dexCachePath) { try { // Download project files into a temporary directory File projectRoot = createNewTempDir(); LOG.info("temporary project root: " + projectRoot.getAbsolutePath()); try { List<String> sourceFiles; try { sourceFiles = extractProjectFiles(inputZip, projectRoot); } catch (IOException e) { LOG.severe("unexpected problem extracting project file from zip"); return Result.createFailingResult("", "Problems processing zip file."); } try { genYailFilesIfNecessary(sourceFiles); } catch (YailGenerationException e) { // Note that we're using a special result code here for the case of a Yail gen error. return new Result(Result.YAIL_GENERATION_ERROR, "", e.getMessage(), e.getFormName()); } catch (Exception e) { LOG.severe("Unknown exception signalled by genYailFilesIf Necessary"); e.printStackTrace(); return Result.createFailingResult("", "Unexpected problems generating YAIL."); } File keyStoreFile = new File(projectRoot, KEYSTORE_FILE_NAME); String keyStorePath = keyStoreFile.getPath(); if (!keyStoreFile.exists()) { keyStorePath = createKeyStore(userName, projectRoot, KEYSTORE_FILE_NAME); saveKeystore = true; } // Create project object from project properties file. Project project = getProjectProperties(projectRoot); File buildTmpDir = new File(projectRoot, "build/tmp"); buildTmpDir.mkdirs(); // Prepare for redirection of compiler message output ByteArrayOutputStream output = new ByteArrayOutputStream(); PrintStream console = new PrintStream(output); ByteArrayOutputStream errors = new ByteArrayOutputStream(); PrintStream userErrors = new PrintStream(errors); Set<String> componentTypes = (isForRepl || isForWireless) ? getAllComponentTypes() : getComponentTypes(sourceFiles); // Invoke YoungAndroid compiler boolean success = Compiler.compile(project, componentTypes, console, console, userErrors, isForRepl, isForWireless, keyStorePath, childProcessRam, dexCachePath); console.close(); userErrors.close(); // Retrieve compiler messages and convert to HTML and log String srcPath = projectRoot.getAbsolutePath() + "/" + PROJECT_DIRECTORY + "/../src/"; String messages = processCompilerOutput(output.toString(PathUtil.DEFAULT_CHARSET), srcPath); if (success) { // Locate output file File outputFile = new File(projectRoot, "build/deploy/" + project.getProjectName() + ".apk"); if (!outputFile.exists()) { LOG.warning("Young Android build - " + outputFile + " does not exist"); } else { outputApk = new File(outputDir, outputFile.getName()); Files.copy(outputFile, outputApk); if (saveKeystore) { outputKeystore = new File(outputDir, KEYSTORE_FILE_NAME); Files.copy(keyStoreFile, outputKeystore); } } } return new Result(success, messages, errors.toString(PathUtil.DEFAULT_CHARSET)); } finally { // On some platforms (OS/X), the java.io.tmpdir contains a symlink. We need to use the // canonical path here so that Files.deleteRecursively will work. // Note (ralph): deleteRecursively has been removed from the guava-11.0.1 lib // Replacing with deleteDirectory, which is supposed to delete the entire directory. FileUtils.deleteDirectory(new File(projectRoot.getCanonicalPath())); } } catch (Exception e) { e.printStackTrace(); return Result.createFailingResult("", "Server error performing build"); } } private void genYailFilesIfNecessary(List<String> sourceFiles) throws IOException, YailGenerationException { // Filter out the files that aren't really source files (i.e. that don't end in .scm or .yail) Collection<String> formAndYailSourceFiles = Collections2.filter(sourceFiles, new Predicate<String>() { @Override public boolean apply(String input) { return input.endsWith(FORM_PROPERTIES_EXTENSION) || input.endsWith(YAIL_EXTENSION); } }); for (String sourceFile : formAndYailSourceFiles) { if (sourceFile.endsWith(FORM_PROPERTIES_EXTENSION)) { String rootPath = sourceFile.substring(0, sourceFile.length() - FORM_PROPERTIES_EXTENSION.length()); String yailFilePath = rootPath + YAIL_EXTENSION; // Note: Famous last words: The following contains() makes this method O(n**2) but n should // be pretty small. if (!sourceFiles.contains(yailFilePath)) { generateYail(rootPath); } } } } private static Set<String> getAllComponentTypes() throws IOException { Set<String> compSet = Sets.newHashSet(); String[] components = Resources .toString(ProjectBuilder.class.getResource(ALL_COMPONENT_TYPES), Charsets.UTF_8).split("\n"); for (String component : components) { compSet.add(component); } return compSet; } private ArrayList<String> extractProjectFiles(ZipFile inputZip, File projectRoot) throws IOException { ArrayList<String> projectFileNames = Lists.newArrayList(); Enumeration<? extends ZipEntry> inputZipEnumeration = inputZip.entries(); while (inputZipEnumeration.hasMoreElements()) { ZipEntry zipEntry = inputZipEnumeration.nextElement(); final InputStream extractedInputStream = inputZip.getInputStream(zipEntry); File extractedFile = new File(projectRoot, zipEntry.getName()); LOG.info("extracting " + extractedFile.getAbsolutePath() + " from input zip"); Files.createParentDirs(extractedFile); // Do I need this? Files.copy(new InputSupplier<InputStream>() { public InputStream getInput() throws IOException { return extractedInputStream; } }, extractedFile); projectFileNames.add(extractedFile.getPath()); } return projectFileNames; } private static Set<String> getComponentTypes(List<String> files) throws IOException { Set<String> componentTypes = Sets.newHashSet(); for (String f : files) { if (f.endsWith(".scm")) { File scmFile = new File(f); String scmContent = new String(Files.toByteArray(scmFile), PathUtil.DEFAULT_CHARSET); componentTypes.addAll(getTypesFromScm(scmContent)); } } return componentTypes; } static String createKeyStore(String userName, File projectRoot, String keystoreFileName) throws IOException { File keyStoreFile = new File(projectRoot.getPath(), keystoreFileName); /* Note: must expire after October 22, 2033, to be in the Android * marketplace. Android docs recommend "10000" as the expiration # of * days. * * For DNAME, US may not the right country to assign it to. */ String[] keytoolCommandline = { System.getProperty("java.home") + "/bin/keytool", "-genkey", "-keystore", keyStoreFile.getAbsolutePath(), "-alias", "AndroidKey", "-keyalg", "RSA", "-dname", "CN=" + quotifyUserName(userName) + ", O=AppInventor for Android, C=US", "-validity", "10000", "-storepass", "android", "-keypass", "android" }; if (Execution.execute(null, keytoolCommandline, System.out, System.err)) { if (keyStoreFile.length() > 0) { return keyStoreFile.getAbsolutePath(); } } return null; } @VisibleForTesting static Set<String> getTypesFromScm(String scm) { return FormPropertiesAnalyzer.getComponentTypesFromFormFile(scm); } @VisibleForTesting static String processCompilerOutput(String output, String srcPath) { // First, remove references to the temp source directory from the messages. String messages = output.replace(srcPath, ""); // Then, format warnings and errors nicely. try { // Split the messages by \n and process each line separately. String[] lines = messages.split("\n"); Pattern pattern = Pattern.compile("(.*?):(\\d+):\\d+: (error|warning)?:? ?(.*?)"); StringBuilder sb = new StringBuilder(); boolean skippedErrorOrWarning = false; for (String line : lines) { Matcher matcher = pattern.matcher(line); if (matcher.matches()) { // Determine whether it is an error or warning. String kind; String spanClass; // Scanner messages do not contain either 'error' or 'warning'. // I treat them as errors because they prevent compilation. if ("warning".equals(matcher.group(3))) { kind = "WARNING"; spanClass = "compiler-WarningMarker"; } else { kind = "ERROR"; spanClass = "compiler-ErrorMarker"; } // Extract the filename, lineNumber, and message. String filename = matcher.group(1); String lineNumber = matcher.group(2); String text = matcher.group(4); // If the error/warning is in a yail file, generate a div and append it to the // StringBuilder. if (filename.endsWith(YoungAndroidConstants.YAIL_EXTENSION)) { skippedErrorOrWarning = false; sb.append("<div><span class='" + spanClass + "'>" + kind + "</span>: " + StringUtils.escape(filename) + " line " + lineNumber + ": " + StringUtils.escape(text) + "</div>"); } else { // The error/warning is in runtime.scm. Don't append it to the StringBuilder. skippedErrorOrWarning = true; } // Log the message, first truncating it if it is too long. if (text.length() > MAX_COMPILER_MESSAGE_LENGTH) { text = text.substring(0, MAX_COMPILER_MESSAGE_LENGTH); } } else { // The line isn't an error or a warning. This is expected. // If the line begins with two spaces, it is a continuation of the previous // error/warning. if (line.startsWith(" ")) { // If we didn't skip the most recent error/warning, append the line to our // StringBuilder. if (!skippedErrorOrWarning) { sb.append(StringUtils.escape(line)).append("<br>"); } } else { skippedErrorOrWarning = false; // We just append the line to our StringBuilder. sb.append(StringUtils.escape(line)).append("<br>"); } } } messages = sb.toString(); } catch (Exception e) { // Report exceptions that happen during the processing of output, but don't make the // whole build fail. e.printStackTrace(); // We were not able to process the output, so we just escape for HTML. messages = StringUtils.escape(messages); } return messages; } /* * Adds quotes around the given userName and encodes embedded quotes as \". */ private static String quotifyUserName(String userName) { Preconditions.checkNotNull(userName); int length = userName.length(); StringBuilder sb = new StringBuilder(length + 2); sb.append('"'); for (int i = 0; i < length; i++) { char ch = userName.charAt(i); if (ch == '"') { sb.append('\\').append(ch); } else { sb.append(ch); } } sb.append('"'); return sb.toString(); } /* * Loads the project properties file of a Young Android project. */ private Project getProjectProperties(File projectRoot) { return new Project(projectRoot.getAbsolutePath() + "/" + PROJECT_PROPERTIES_FILE_NAME); } private File generateYail(String rootName) throws IOException, YailGenerationException { String formPropertiesPath = rootName + FORM_PROPERTIES_EXTENSION; String codeblocksSourcePath = rootName + CODEBLOCKS_SOURCE_EXTENSION; String yailPath = rootName + YAIL_EXTENSION; String[] commandLine = { System.getProperty("java.home") + "/bin/java", "-mx1024M", "-jar", Compiler.getResource(Compiler.RUNTIME_FILES_DIR + "YailGenerator.jar"), new File(formPropertiesPath).getAbsolutePath(), new File(codeblocksSourcePath).getAbsolutePath(), yailPath }; StringBuffer out = new StringBuffer(); StringBuffer err = new StringBuffer(); int exitValue = Execution.execute(null, commandLine, out, err); if (exitValue == 0) { String generatedYailString = out.toString(); File generatedYailFile = new File(yailPath); Files.write(generatedYailString, generatedYailFile, Charsets.UTF_8); return generatedYailFile; } else { String formName = PathUtil.trimOffExtension(PathUtil.basename(formPropertiesPath)); if (exitValue == 1) { // Failed to generate yail for legitimate reasons, such as empty sockets. throw new YailGenerationException("Unable to generate code for " + formName + "." + "\n -- err is " + err.toString() + "\n -- out is" + out.toString(), formName); } else { // Any other exit value is unexpected. throw new RuntimeException("YailGenerator for form " + formName + " exited with code " + exitValue + "\n -- err is " + err.toString() + "\n -- out is" + out.toString()); } } } private static class YailGenerationException extends Exception { // The name of the form being built when an error occurred private final String formName; YailGenerationException(String message, String formName) { super(message); this.formName = formName; } /** * Return the name of the form that Yail generation failed on. */ String getFormName() { return formName; } } public int getProgress() { return Compiler.getProgress(); } }