Java tutorial
package org.apache.gaelucene.tools; /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.LineNumberReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.net.HttpURLConnection; import java.net.URL; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashSet; import org.apache.lucene.index.IndexReader; import org.apache.lucene.store.GAEFile; public class LuceneIndexPushUtil { //private static final String GAE_APP_URL = "http://3gmatrix-search.appspot.com/dashboard/gaelucene"; private static final String GAE_APP_URL = "http://localhost:8080/gaelucenedashboard"; private static boolean commitFile(File file, String category, long version, int fi) throws IOException { String fileName = file.getName(); long fileLength = file.length(); long lastModified = file.lastModified(); System.out.println("file[" + fi + "].fileName:" + fileName); System.out.println("file[" + fi + "].length:" + fileLength); System.out.println("file[" + fi + "].lastModified:" + (dateFormat.format(new Date(lastModified)))); int segmentCount = (int) (fileLength + 1) / GAEFile.SEGMENT_LENGTH + 1; long fileId = -1; { StringBuffer requestUrl = new StringBuffer(128); requestUrl.append(gaeAppURL); requestUrl.append("/registernewindexfile?"); requestUrl.append("&cat=").append(category); requestUrl.append("&ver=").append(version); requestUrl.append("&name=").append(fileName); requestUrl.append("&length=").append(fileLength); requestUrl.append("&lastModified=").append(lastModified); requestUrl.append("&segmentCount=").append(segmentCount); URL url = new URL(requestUrl.toString()); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty("Cookie", authCookie); conn.setDoOutput(true); conn.connect(); OutputStream os = conn.getOutputStream(); os.write(0); os.flush(); os.close(); //====================================================================== System.out.println("registering '" + fileName + "' ...."); BufferedInputStream in = new BufferedInputStream(conn.getInputStream()); ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] bytes = new byte[1024]; int length = 0; for (int i = in.read(bytes); i != -1; i = in.read(bytes)) { out.write(bytes, 0, i); length += i; } in.close(); in = null; conn.disconnect(); conn = null; byte[] ostream = out.toByteArray(); String content = new String(ostream, "UTF-8"); System.out.println("response:" + content); fileId = Long.parseLong(content); System.out.println("'" + fileName + "' registered, fileId:" + fileId + "|"); } //====================================================================== System.out.print("uploading '" + fileName + "' ...."); FileInputStream fis = new FileInputStream(file); long bytesUnread = fileLength; for (int i = 0; i < segmentCount; i++) { System.out.print("\nuploading '" + fileName + "[" + i + "]' ...."); StringBuffer requestUrl = new StringBuffer(128); requestUrl.append(gaeAppURL); requestUrl.append("/commitnewindexfile?"); requestUrl.append("&fileId=").append(fileId); requestUrl.append("&segmentNo=").append(i); requestUrl.append("&segmentLength=").append(Math.min(bytesUnread, GAEFile.SEGMENT_LENGTH)); URL url = new URL(requestUrl.toString()); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty("Cookie", authCookie); conn.setDoOutput(true); conn.connect(); OutputStream os = conn.getOutputStream(); byte[] buffer = new byte[GAEFile.SEGMENT_LENGTH]; int bytesNeedRead = GAEFile.SEGMENT_LENGTH; while (bytesNeedRead > 0) { int bytes = fis.read(buffer, 0, bytesNeedRead); if (bytes < 0) { break; } System.out.print("."); os.write(buffer, 0, bytes); bytesNeedRead -= bytes; } os.flush(); os.close(); BufferedInputStream in = new BufferedInputStream(conn.getInputStream()); ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] bytes = new byte[1024]; int length = 0; for (int c = in.read(bytes); c != -1; c = in.read(bytes)) { out.write(bytes, 0, c); length += c; } in.close(); in = null; conn.disconnect(); conn = null; byte[] ostream = out.toByteArray(); String content = new String(ostream, "UTF-8"); System.out.println("\nresponse:" + content); } fis.close(); System.out.println("fine! over!\n\n"); return true; } private static boolean activateIndex(String category, long version) throws IOException { StringBuffer requestUrl = new StringBuffer(128); requestUrl.append(gaeAppURL); requestUrl.append("/activatenewindex?"); requestUrl.append("&cat=").append(category); requestUrl.append("&ver=").append(version); URL url = new URL(requestUrl.toString()); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty("Cookie", authCookie); conn.setDoOutput(true); conn.connect(); System.out.print("activating...."); OutputStream os = conn.getOutputStream(); byte[] buffer = new byte[] { 0 }; os.write(buffer, 0, buffer.length); os.flush(); os.close(); //====================================================================== BufferedInputStream in = new BufferedInputStream(conn.getInputStream()); ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] bytes = new byte[1024]; int length = 0; for (int i = in.read(bytes); i != -1; i = in.read(bytes)) { out.write(bytes, 0, i); length += i; } in.close(); in = null; byte[] ostream = out.toByteArray(); String content = new String(ostream, "UTF-8"); System.out.println("fine! over!"); System.out.println("response:" + content); return true; } private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private static final String USAGE = "java " + LuceneIndexPushUtil.class.getName() + " -app-url ${gaeAppURL} -auth-cookie ${authCookie} -cat ${index-category} -src ${path-to-index-folder} -rec-file ${path-to-rec-file}"; // app url, eg: http://gaelucene.appspot.com/gaelucenedashboard static String gaeAppURL = GAE_APP_URL; // use cookie to act as authenticated user static String authCookie = null; // index source path static String sourceDirName = null; // index category static String category = null; // record filenames that have been uploaded successfully use jobRecFile, // with the jobRecFile, you can retry some individual file that was failed to upload. static String jobRecFileName = null; public static void main(String[] args) throws IOException { for (int i = 0; i < args.length; i++) { if ("-app-url".equals(args[i])) { gaeAppURL = args[++i]; } else if ("-auth-cookie".equals(args[i])) { authCookie = args[++i]; } else if ("-src".equals(args[i])) { sourceDirName = args[++i]; } else if ("-cat".equals(args[i])) { category = args[++i]; } else if ("-rec-file".equals(args[i])) { jobRecFileName = args[++i]; } } if (gaeAppURL == null || authCookie == null || sourceDirName == null || category == null || jobRecFileName == null) { System.err.println(USAGE); System.exit(-1); } File sourceDir = new File(sourceDirName); if (!sourceDir.exists()) { System.err.println("'" + sourceDir.getAbsolutePath() + "' DOES NOT EXIST!"); System.exit(-1); } sourceDirName = sourceDir.getAbsolutePath(); // load filenames that have been uploaded successfully last time. HashSet<String> uploadedRec = new HashSet<String>(); File jobRecFile = new File(jobRecFileName); if (jobRecFile.exists()) { LineNumberReader reader = new LineNumberReader(new FileReader(jobRecFile)); for (String line = reader.readLine(); line != null;) { if (line.indexOf(" OK") > -1) { line = line.substring(0, line.indexOf(" ")).trim(); } uploadedRec.add(line); line = reader.readLine(); } reader.close(); } System.out.println("[INFO ] - trying to open index under " + sourceDirName); IndexReader indexReader = IndexReader.open(sourceDir); int maxDoc = indexReader.maxDoc(); int numDocs = indexReader.numDocs(); long version = indexReader.getVersion(); boolean hasDeletions = indexReader.hasDeletions(); boolean isOptimized = indexReader.isOptimized(); System.out.println("maxDoc:" + maxDoc); System.out.println("numDocs:" + numDocs); System.out.println("version:" + version); System.out.println("hasDeletions:" + hasDeletions); System.out.println("isOptimized:" + isOptimized); // record filenames that were uploaded successfully BufferedWriter dataWriter = new BufferedWriter( new OutputStreamWriter(new FileOutputStream(jobRecFile, true))); System.out.println("[INFO ] - trying to synchronize the index files onto gae..."); File[] files = sourceDir.listFiles(); for (int i = 0; i < files.length; i++) { File file = files[i]; if (uploadedRec.contains(file.getName())) { System.out.println("[INFO ] - skip file '" + file.getName() + "'"); continue; } try { commitFile(file, category, version, i); dataWriter.write(file.getName() + " OK\n"); } catch (IOException ioe) { System.out.println("[WARN ] - failed to upload '" + file.getName() + "', because:" + ioe); } } dataWriter.flush(); dataWriter.close(); System.out.println("[INFO ] - trying to activate the index..."); try { activateIndex(category, version); } catch (IOException ioe) { System.out.println("[WARN ] - failed to activate the index, because:" + ioe); } } }