Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.aliyun.odps.local.common.utils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileFilter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.Charset; import java.security.AccessController; import java.security.PrivilegedAction; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.SimpleTimeZone; import javax.mail.internet.MimeUtility; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.HiddenFileFilter; import org.apache.commons.io.filefilter.IOFileFilter; import com.aliyun.odps.Column; import com.aliyun.odps.OdpsType; import com.aliyun.odps.Table; import com.aliyun.odps.local.common.Constants; import com.aliyun.odps.local.common.ExceptionCode; import com.aliyun.odps.local.common.TableMeta; import com.aliyun.odps.local.common.WareHouse; public class LocalRunUtils { public static long getPID() { String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); return Long.parseLong(processName.split("@")[0]); } public static String getCurrentTime() { return getDateFormat(Constants.DATE_FORMAT_2).format(new Date()); } public static void sleep(long millis) { try { Thread.sleep(millis); } catch (InterruptedException e) { } } public static void checkArgument(String name, int value, int lower_bound, int upper_bound) { if (value < lower_bound || value > upper_bound) { throw new RuntimeException("Local Run: Value of " + name + " out of bound, must be in range [" + lower_bound + "," + upper_bound + "]."); } } public static Object fromString(OdpsType type, String val, String nullIndicator, boolean isBinary) throws IOException { if (val == null) { return null; } if (nullIndicator != null && nullIndicator.equals(val)) { return null; } if (val != null) { switch (type) { case BIGINT: return Long.parseLong(val); case STRING: try { byte[] v = fromReadableString(val); return isBinary ? v : new String(v, Charset.forName("UTF-8")); } catch (Exception e1) { throw new RuntimeException("from readable string failed!" + e1); } case DOUBLE: return Double.parseDouble(val); case BOOLEAN: return Boolean.parseBoolean(val); case DATETIME: try { return getDateFormat(Constants.DATE_FORMAT_2).parse(val); } catch (ParseException e) { throw new IOException(e); } default: throw new IOException("unsupported type: " + type); } } else { return null; } } public static Object fromString(OdpsType type, String val, String nullIndicator) throws IOException { return fromString(type, val, nullIndicator, false); } public static DateFormat getDateFormat(String formateStr) { SimpleDateFormat dateFormat = new SimpleDateFormat(formateStr); return dateFormat; } /** * ????? */ public static List<File> listEmptyDirectory(File dir) { List<File> dataFiles = new ArrayList<File>(); File[] subDirs = dir.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname.isDirectory() && !pathname.isHidden(); } }); for (File file : subDirs) { if (file.isDirectory()) { listEmptyDirectory(file, dataFiles); } } return dataFiles; } private static void listEmptyDirectory(File dir, List<File> dataFiles) { if (!dir.isDirectory()) { return; } File[] subDirs = dir.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return !pathname.isHidden(); } }); if (subDirs.length == 0) { System.out.println(dir.getAbsolutePath()); dataFiles.add(dir); return; } for (File f : subDirs) { if (f.isDirectory()) { listEmptyDirectory(f, dataFiles); } } } /** * ?????? * * @param dir * @return */ public static List<File> listDataFiles(File dir) { List<File> dataFiles = new ArrayList<File>(); for (File file : dir.listFiles()) { if (!Constants.SCHEMA_FILE.equals(file.getName()) && file.isFile()) { dataFiles.add(file); } } return dataFiles; } /** * ????? * * @param srcFile * @param result */ public static void listAllDataFiles(File srcFile, List<File> result) { if (srcFile.isDirectory()) { final Collection<File> list = FileUtils.listFiles(srcFile, HiddenFileFilter.VISIBLE, HiddenFileFilter.VISIBLE); for (File f : list) { listAllDataFiles(f, result); } } else if (!Constants.SCHEMA_FILE.equals(srcFile.getName())) { result.add(srcFile); } } public static void removeDataFiles(File dir) { for (File file : listDataFiles(dir)) { file.delete(); } } public static List<Integer> genReadColsIndexes(TableMeta meta, String[] readCols) { if (readCols == null || readCols.length == 0 || readCols[0] == null || readCols[0].trim().length() == 0) { return null; } List<Integer> indexes = new ArrayList<Integer>(); for (String col : readCols) { boolean found = false; for (int j = 0; j < meta.getCols().length; ++j) { if (meta.getCols()[j].getName().equals(col)) { indexes.add(j); found = true; break; } } if (!found) { throw new RuntimeException(ExceptionCode.ODPS_0720021 + " - " + col + " in table " + meta.getProjName() + "." + meta.getTableName()); } } return indexes; } public static Column[] getInputTableFields(TableMeta meta, String[] cols) { if (cols == null || cols[0].length() == 0) { return meta.getCols(); } List<Column> fields = new ArrayList<Column>(); for (String col : cols) { for (Column field : meta.getCols()) { if (col.equals(field.getName())) { fields.add(field); } } } return fields.toArray(new Column[fields.size()]); } public static Column[] getInputTableFields(Table table, String[] cols) { List<Column> schema = table.getSchema().getColumns(); if (cols == null || cols.length == 0) { return null; } Column[] fields = new Column[cols.length]; for (int i = 0; i < cols.length; i++) { for (Column field : schema) { if (schema.get(i).getName().equals(field.getName())) { fields[i] = schema.get(i); } } } return fields; } public static List<String> parseResourceName(String resourceName, String defaultProject) throws IOException { List<String> ret = new ArrayList<String>(); resourceName = resourceName.trim(); if (resourceName.isEmpty()) { throw new IllegalArgumentException(ExceptionCode.ODPS_0720031); } String[] arr = resourceName.split("/"); if (arr.length == 1) { ret.add(defaultProject); ret.add(arr[0]); } else if (arr.length == 3) { if (!arr[1].equals("resources")) { throw new IllegalArgumentException(ExceptionCode.ODPS_0720041 + " - '" + resourceName + "'"); } ret.add(arr[0]); ret.add(arr[2]); } else { throw new IllegalArgumentException(ExceptionCode.ODPS_0720041 + " - '" + resourceName + "'"); } return ret; } public static String getDownloadErrorMsg(String tableInfo) { return "\nTable '" + tableInfo + "' is not exists in local warehouse, and download failed! Please check the following information:\n" + "1. Endpoint/accessId/accessKey\n" + "2. Project/Table/Partition name\n" + "3. Local warehouse table __schema__ file infomation.\n" + "4. If remote server has this table.\n" + "Download Mode:" + WareHouse.getInstance().getDownloadMode().toString(); } /** * relativePath?parent??? * CanoncialFile???????? * child??getParentFile?parent???? * child???null * * * @param parent * * @param realativePath * ? * @throws IOException * ?????? */ private static void checkParent(File parent, String relativePath) { try { File child = new File(parent, relativePath); if (!child.exists()) { throw new IOException("ODPS-0140171: must set a correct realtive path:" + relativePath); } while (child != null && !child.getCanonicalFile().equals(parent.getCanonicalFile())) { if (FileUtils.isSymlink(child)) { throw new IOException("ODPS-0140171: not allow symlink in archive files:" + child.getName()); } child = child.getParentFile(); } if (child == null) { throw new IOException("ODPS-0140171: not correct parameter of relative path in getCacheArchive... :" + relativePath); } } catch (IOException e) { throw new RuntimeException(e); } catch (SecurityException e) { IOException tmp_e = new IOException("ODPS-0140171: permission denied to read archive resource '" + parent.getName() + "' with relative path :" + relativePath); throw new RuntimeException(tmp_e); } } @SuppressWarnings({ "unchecked", "rawtypes" }) public static Collection<File> listFiles(final File dir, final String relativePath) throws IOException { // privileged code, for this method may be invoked by user code try { return (Collection<File>) AccessController.doPrivileged(new PrivilegedAction() { public Object run() { File relativeDir = new File(dir, relativePath); checkParent(dir, relativePath); if (relativeDir.isDirectory()) { return FileUtils.listFiles(relativeDir, new InternalIOFilter(), new InternalIOFilter()); } else { Collection<File> files = new java.util.LinkedList<File>(); files.add(relativeDir); return files; } } }); } catch (RuntimeException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } else { throw e; } } } private static class InternalIOFilter implements IOFileFilter { public InternalIOFilter() { super(); } @Override public boolean accept(File dir, String name) { return accept(new File(dir, name)); } @Override public boolean accept(File file) { try { if (FileUtils.isSymlink(file)) { throw new IOException("ODPS-0140171: not allow symlink in archive files:" + file.getName()); } return true; } catch (IOException e) { throw new RuntimeException(e); } } } public static String toReadableString(byte[] b) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream printableos = MimeUtility.encode(baos, "quoted-printable"); printableos.write(b); printableos.close(); return new String(baos.toByteArray(), Charset.forName("UTF-8")); } public static byte[] fromReadableString(String str) throws Exception { byte[] b = str.getBytes(); ByteArrayInputStream bais = new ByteArrayInputStream(b); InputStream printableis = MimeUtility.decode(bais, "quoted-printable"); byte[] tmp = new byte[b.length]; int n = printableis.read(tmp); byte[] res = new byte[n]; System.arraycopy(tmp, 0, res, 0, n); return res; } }