Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.blur.manager.writer; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.SequenceFile.Reader; import org.apache.hadoop.io.SequenceFile.Writer; import org.apache.hadoop.io.Text; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; public class SnapshotIndexDeletionPolicy extends IndexDeletionPolicy { private static final Log LOG = LogFactory.getLog(SnapshotIndexDeletionPolicy.class); private final Configuration _configuration; private final Path _path; private final Map<String, Long> _namesToGenerations = new ConcurrentHashMap<String, Long>(); private final Map<Long, Set<String>> _generationsToNames = new ConcurrentHashMap<Long, Set<String>>(); public SnapshotIndexDeletionPolicy(Configuration configuration, Path path) throws IOException { _configuration = configuration; _path = path; FileSystem fileSystem = _path.getFileSystem(configuration); fileSystem.mkdirs(path); loadGenerations(); } @Override public void onInit(List<? extends IndexCommit> commits) throws IOException { onCommit(commits); } @Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { int size = commits.size(); for (int i = 0; i < size - 1; i++) { IndexCommit indexCommit = commits.get(i); long generation = indexCommit.getGeneration(); if (!_generationsToNames.containsKey(generation)) { indexCommit.delete(); } } } private synchronized void storeGenerations() throws IOException { FileSystem fileSystem = _path.getFileSystem(_configuration); FileStatus[] listStatus = fileSystem.listStatus(_path); SortedSet<FileStatus> existing = new TreeSet<FileStatus>(Arrays.asList(listStatus)); long currentFile; if (!existing.isEmpty()) { FileStatus last = existing.last(); currentFile = Long.parseLong(last.getPath().getName()); } else { currentFile = 0; } Path path = new Path(_path, buffer(currentFile + 1)); LOG.info("Creating new snapshot file [{0}]", path); FSDataOutputStream outputStream = fileSystem.create(path, false); Writer writer = SequenceFile.createWriter(_configuration, outputStream, Text.class, LongWritable.class, CompressionType.NONE, null); for (Entry<String, Long> e : _namesToGenerations.entrySet()) { writer.append(new Text(e.getKey()), new LongWritable(e.getValue())); } writer.close(); outputStream.close(); cleanupOldFiles(fileSystem, existing); } private void cleanupOldFiles(FileSystem fileSystem, SortedSet<FileStatus> existing) throws IOException { for (FileStatus fileStatus : existing) { fileSystem.delete(fileStatus.getPath(), false); } } private String buffer(long number) { String s = Long.toString(number); StringBuilder builder = new StringBuilder(); for (int i = s.length(); i < 12; i++) { builder.append('0'); } return builder.append(s).toString(); } private void loadGenerations() throws IOException { FileSystem fileSystem = _path.getFileSystem(_configuration); FileStatus[] listStatus = fileSystem.listStatus(_path); SortedSet<FileStatus> existing = new TreeSet<FileStatus>(Arrays.asList(listStatus)); if (existing.isEmpty()) { return; } FileStatus last = existing.last(); Reader reader = new SequenceFile.Reader(fileSystem, last.getPath(), _configuration); Text key = new Text(); LongWritable value = new LongWritable(); while (reader.next(key, value)) { String name = key.toString(); long gen = value.get(); _namesToGenerations.put(name, gen); Set<String> names = _generationsToNames.get(gen); if (names == null) { names = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); _generationsToNames.put(gen, names); } names.add(name); } reader.close(); existing.remove(last); cleanupOldFiles(fileSystem, existing); } public void createSnapshot(String name, DirectoryReader reader, String context) throws IOException { if (_namesToGenerations.containsKey(name)) { throw new IOException("Snapshot [" + name + "] already exists."); } LOG.info("Creating snapshot [{0}] in [{1}].", name, context); IndexCommit indexCommit = reader.getIndexCommit(); long generation = indexCommit.getGeneration(); _namesToGenerations.put(name, generation); Set<String> names = _generationsToNames.get(generation); if (names == null) { names = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); _generationsToNames.put(generation, names); } names.add(name); storeGenerations(); } public void removeSnapshot(String name, String context) throws IOException { Long gen = _namesToGenerations.get(name); if (gen == null) { LOG.info("Snapshot [{0}] does not exist in [{1}].", name, context); return; } LOG.info("Removing snapshot [{0}] from [{1}].", name, context); _namesToGenerations.remove(name); Set<String> names = _generationsToNames.get(gen); names.remove(name); if (names.isEmpty()) { _generationsToNames.remove(gen); } storeGenerations(); } public Collection<String> getSnapshots() { return new HashSet<String>(_namesToGenerations.keySet()); } public Path getSnapshotsDirectoryPath() { return _path; } public Long getGeneration(String name) { return _namesToGenerations.get(name); } public static Path getGenerationsPath(Path shardDir) { return new Path(shardDir, "generations"); } }