Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.core.snapshots; import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.lang.invoke.MethodHandles; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; import org.apache.hadoop.fs.Path; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; /** * This class provides utility functions required for Solr snapshots functionality. */ public class SolrSnapshotsTool implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final DateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault()); private static final String CREATE = "create"; private static final String DELETE = "delete"; private static final String LIST = "list"; private static final String DESCRIBE = "describe"; private static final String PREPARE_FOR_EXPORT = "prepare-snapshot-export"; private static final String EXPORT_SNAPSHOT = "export"; private static final String HELP = "help"; private static final String COLLECTION = "c"; private static final String TEMP_DIR = "t"; private static final String DEST_DIR = "d"; private static final String SOLR_ZK_ENSEMBLE = "z"; private static final String HDFS_PATH_PREFIX = "p"; private static final String BACKUP_REPO_NAME = "r"; private static final String ASYNC_REQ_ID = "i"; private static final List<String> OPTION_HELP_ORDER = Arrays.asList(CREATE, DELETE, LIST, DESCRIBE, PREPARE_FOR_EXPORT, EXPORT_SNAPSHOT, HELP, SOLR_ZK_ENSEMBLE, COLLECTION, DEST_DIR, BACKUP_REPO_NAME, ASYNC_REQ_ID, TEMP_DIR, HDFS_PATH_PREFIX); private final CloudSolrClient solrClient; public SolrSnapshotsTool(String solrZkEnsemble) { solrClient = (new CloudSolrClient.Builder()).withZkHost(solrZkEnsemble).build(); } @Override public void close() throws IOException { if (solrClient != null) { solrClient.close(); } } public void createSnapshot(String collectionName, String snapshotName) { CollectionAdminRequest.CreateSnapshot createSnap = new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName); CollectionAdminResponse resp; try { resp = createSnap.process(solrClient); Preconditions.checkState(resp.getStatus() == 0, "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus()); System.out.println("Successfully created snapshot with name " + snapshotName + " for collection " + collectionName); } catch (Exception e) { log.error("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName, e); System.out.println("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName + " due to following error : " + e.getLocalizedMessage()); } } public void deleteSnapshot(String collectionName, String snapshotName) { CollectionAdminRequest.DeleteSnapshot deleteSnap = new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName); CollectionAdminResponse resp; try { resp = deleteSnap.process(solrClient); Preconditions.checkState(resp.getStatus() == 0, "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus()); System.out.println("Successfully deleted snapshot with name " + snapshotName + " for collection " + collectionName); } catch (Exception e) { log.error("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName, e); System.out.println("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName + " due to following error : " + e.getLocalizedMessage()); } } @SuppressWarnings("rawtypes") public void listSnapshots(String collectionName) { CollectionAdminRequest.ListSnapshots listSnaps = new CollectionAdminRequest.ListSnapshots(collectionName); CollectionAdminResponse resp; try { resp = listSnaps.process(solrClient); Preconditions.checkState(resp.getStatus() == 0, "The LISTSNAPSHOTS request failed. The status code is " + resp.getStatus()); NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO); for (int i = 0; i < apiResult.size(); i++) { System.out.println(apiResult.getName(i)); } } catch (Exception e) { log.error("Failed to list snapshots for collection " + collectionName, e); System.out.println("Failed to list snapshots for collection " + collectionName + " due to following error : " + e.getLocalizedMessage()); } } public void describeSnapshot(String collectionName, String snapshotName) { try { Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName); for (CollectionSnapshotMetaData m : snaps) { if (snapshotName.equals(m.getName())) { System.out.println("Name: " + m.getName()); System.out.println("Status: " + m.getStatus()); System.out.println("Time of creation: " + dateFormat.format(m.getCreationDate())); System.out.println("Total number of cores with snapshot: " + m.getReplicaSnapshots().size()); System.out.println("-----------------------------------"); for (CoreSnapshotMetaData n : m.getReplicaSnapshots()) { StringBuilder builder = new StringBuilder(); builder.append("Core [name="); builder.append(n.getCoreName()); builder.append(", leader="); builder.append(n.isLeader()); builder.append(", generation="); builder.append(n.getGenerationNumber()); builder.append(", indexDirPath="); builder.append(n.getIndexDirPath()); builder.append("]\n"); System.out.println(builder.toString()); } } } } catch (Exception e) { log.error("Failed to fetch snapshot details", e); System.out.println( "Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage()); } } public Map<String, List<String>> getIndexFilesPathForSnapshot(String collectionName, String snapshotName, Optional<String> pathPrefix) throws SolrServerException, IOException { Map<String, List<String>> result = new HashMap<>(); Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName); Optional<CollectionSnapshotMetaData> meta = Optional.empty(); for (CollectionSnapshotMetaData m : snaps) { if (snapshotName.equals(m.getName())) { meta = Optional.of(m); } } if (!meta.isPresent()) { throw new IllegalArgumentException( "The snapshot named " + snapshotName + " is not found for collection " + collectionName); } DocCollection collectionState = solrClient.getZkStateReader().getClusterState() .getCollection(collectionName); for (Slice s : collectionState.getSlices()) { List<CoreSnapshotMetaData> replicaSnaps = meta.get().getReplicaSnapshotsForShard(s.getName()); // Prepare a list of *existing* replicas (since one or more replicas could have been deleted after the snapshot creation). List<CoreSnapshotMetaData> availableReplicas = new ArrayList<>(); for (CoreSnapshotMetaData m : replicaSnaps) { if (isReplicaAvailable(s, m.getCoreName())) { availableReplicas.add(m); } } if (availableReplicas.isEmpty()) { throw new IllegalArgumentException("The snapshot named " + snapshotName + " not found for shard " + s.getName() + " of collection " + collectionName); } // Prefer a leader replica (at the time when the snapshot was created). CoreSnapshotMetaData coreSnap = availableReplicas.get(0); for (CoreSnapshotMetaData m : availableReplicas) { if (m.isLeader()) { coreSnap = m; } } String indexDirPath = coreSnap.getIndexDirPath(); if (pathPrefix.isPresent()) { // If the path prefix is specified, rebuild the path to the index directory. Path t = new Path(coreSnap.getIndexDirPath()); indexDirPath = (new Path(pathPrefix.get(), t.toUri().getPath())).toString(); } List<String> paths = new ArrayList<>(); for (String fileName : coreSnap.getFiles()) { Path p = new Path(indexDirPath, fileName); paths.add(p.toString()); } result.put(s.getName(), paths); } return result; } public void buildCopyListings(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix) throws SolrServerException, IOException { Map<String, List<String>> paths = getIndexFilesPathForSnapshot(collectionName, snapshotName, pathPrefix); for (Map.Entry<String, List<String>> entry : paths.entrySet()) { StringBuilder filesBuilder = new StringBuilder(); for (String filePath : entry.getValue()) { filesBuilder.append(filePath); filesBuilder.append("\n"); } String files = filesBuilder.toString().trim(); try (Writer w = new OutputStreamWriter(new FileOutputStream(new File(localFsPath, entry.getKey())), StandardCharsets.UTF_8)) { w.write(files); } } } public void backupCollectionMetaData(String collectionName, String snapshotName, String backupLoc) throws SolrServerException, IOException { // Backup the collection meta-data CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName); backup.setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY); backup.setLocation(backupLoc); CollectionAdminResponse resp = backup.process(solrClient); Preconditions.checkState(resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus()); } public void prepareForExport(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix, String destPath) { try { buildCopyListings(collectionName, snapshotName, localFsPath, pathPrefix); System.out.println("Successfully prepared copylisting for the snapshot export."); } catch (Exception e) { log.error("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection " + collectionName, e); System.out.println("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection " + collectionName + " due to following error : " + e.getLocalizedMessage()); System.exit(1); } try { backupCollectionMetaData(collectionName, snapshotName, destPath); System.out.println("Successfully backed up collection meta-data"); } catch (Exception e) { log.error("Failed to backup collection meta-data for collection " + collectionName, e); System.out.println("Failed to backup collection meta-data for collection " + collectionName + " due to following error : " + e.getLocalizedMessage()); System.exit(1); } } public void exportSnapshot(String collectionName, String snapshotName, String destPath, Optional<String> backupRepo, Optional<String> asyncReqId) { try { CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName); backup.setCommitName(snapshotName); backup.setIndexBackupStrategy(CollectionAdminParams.COPY_FILES_STRATEGY); backup.setLocation(destPath); if (backupRepo.isPresent()) { backup.setRepositoryName(backupRepo.get()); } if (asyncReqId.isPresent()) { backup.setAsyncId(asyncReqId.get()); } CollectionAdminResponse resp = backup.process(solrClient); Preconditions.checkState(resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus()); } catch (Exception e) { log.error("Failed to backup collection meta-data for collection " + collectionName, e); System.out.println("Failed to backup collection meta-data for collection " + collectionName + " due to following error : " + e.getLocalizedMessage()); System.exit(1); } } public static void main(String[] args) throws IOException { CommandLineParser parser = new PosixParser(); Options options = new Options(); options.addOption(null, CREATE, true, "This command will create a snapshot with the specified name"); options.addOption(null, DELETE, true, "This command will delete a snapshot with the specified name"); options.addOption(null, LIST, false, "This command will list all the named snapshots for the specified collection."); options.addOption(null, DESCRIBE, true, "This command will print details for a named snapshot for the specified collection."); options.addOption(null, PREPARE_FOR_EXPORT, true, "This command will prepare copylistings for the specified snapshot." + " This command should only be used only if Solr is deployed with Hadoop and collection index files are stored on a shared" + " file-system e.g. HDFS"); options.addOption(null, EXPORT_SNAPSHOT, true, "This command will create a backup for the specified snapshot."); options.addOption(null, HELP, false, "This command will print the help message for the snapshots related commands."); options.addOption(TEMP_DIR, true, "This parameter specifies the path of a temporary directory on local filesystem" + " during prepare-snapshot-export command."); options.addOption(DEST_DIR, true, "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related" + " information should be stored."); options.addOption(COLLECTION, true, "This parameter specifies the name of the collection to be used during snapshot operation"); options.addOption(SOLR_ZK_ENSEMBLE, true, "This parameter specifies the Solr Zookeeper ensemble address"); options.addOption(HDFS_PATH_PREFIX, true, "This parameter specifies the HDFS URI prefix to be used" + " during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS."); options.addOption(BACKUP_REPO_NAME, true, "This parameter specifies the name of the backup repository to be used" + " during snapshot export preparation"); options.addOption(ASYNC_REQ_ID, true, "This parameter specifies the async request identifier to be used" + " during snapshot export preparation"); CommandLine cmd = null; try { cmd = parser.parse(options, args); } catch (ParseException e) { System.out.println(e.getLocalizedMessage()); printHelp(options); System.exit(1); } if (cmd.hasOption(CREATE) || cmd.hasOption(DELETE) || cmd.hasOption(LIST) || cmd.hasOption(DESCRIBE) || cmd.hasOption(PREPARE_FOR_EXPORT) || cmd.hasOption(EXPORT_SNAPSHOT)) { try (SolrSnapshotsTool tool = new SolrSnapshotsTool(requiredArg(options, cmd, SOLR_ZK_ENSEMBLE))) { if (cmd.hasOption(CREATE)) { String snapshotName = cmd.getOptionValue(CREATE); String collectionName = requiredArg(options, cmd, COLLECTION); tool.createSnapshot(collectionName, snapshotName); } else if (cmd.hasOption(DELETE)) { String snapshotName = cmd.getOptionValue(DELETE); String collectionName = requiredArg(options, cmd, COLLECTION); tool.deleteSnapshot(collectionName, snapshotName); } else if (cmd.hasOption(LIST)) { String collectionName = requiredArg(options, cmd, COLLECTION); tool.listSnapshots(collectionName); } else if (cmd.hasOption(DESCRIBE)) { String snapshotName = cmd.getOptionValue(DESCRIBE); String collectionName = requiredArg(options, cmd, COLLECTION); tool.describeSnapshot(collectionName, snapshotName); } else if (cmd.hasOption(PREPARE_FOR_EXPORT)) { String snapshotName = cmd.getOptionValue(PREPARE_FOR_EXPORT); String collectionName = requiredArg(options, cmd, COLLECTION); String localFsDir = requiredArg(options, cmd, TEMP_DIR); String hdfsOpDir = requiredArg(options, cmd, DEST_DIR); Optional<String> pathPrefix = Optional.ofNullable(cmd.getOptionValue(HDFS_PATH_PREFIX)); if (pathPrefix.isPresent()) { try { new URI(pathPrefix.get()); } catch (URISyntaxException e) { System.out.println("The specified File system path prefix " + pathPrefix.get() + " is invalid. The error is " + e.getLocalizedMessage()); System.exit(1); } } tool.prepareForExport(collectionName, snapshotName, localFsDir, pathPrefix, hdfsOpDir); } else if (cmd.hasOption(EXPORT_SNAPSHOT)) { String snapshotName = cmd.getOptionValue(EXPORT_SNAPSHOT); String collectionName = requiredArg(options, cmd, COLLECTION); String destDir = requiredArg(options, cmd, DEST_DIR); Optional<String> backupRepo = Optional.ofNullable(cmd.getOptionValue(BACKUP_REPO_NAME)); Optional<String> asyncReqId = Optional.ofNullable(cmd.getOptionValue(ASYNC_REQ_ID)); tool.exportSnapshot(collectionName, snapshotName, destDir, backupRepo, asyncReqId); } } } else if (cmd.hasOption(HELP)) { printHelp(options); } else { System.out.println("Unknown command specified."); printHelp(options); } } private static String requiredArg(Options options, CommandLine cmd, String optVal) { if (!cmd.hasOption(optVal)) { System.out.println("Please specify the value for option " + optVal); printHelp(options); System.exit(1); } return cmd.getOptionValue(optVal); } private static boolean isReplicaAvailable(Slice s, String coreName) { for (Replica r : s.getReplicas()) { if (coreName.equals(r.getCoreName())) { return true; } } return false; } private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(String collectionName) throws SolrServerException, IOException { CollectionAdminRequest.ListSnapshots listSnapshots = new CollectionAdminRequest.ListSnapshots( collectionName); CollectionAdminResponse resp = listSnapshots.process(solrClient); Preconditions.checkState(resp.getStatus() == 0); NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO); Collection<CollectionSnapshotMetaData> result = new ArrayList<>(); for (int i = 0; i < apiResult.size(); i++) { result.add(new CollectionSnapshotMetaData((NamedList<Object>) apiResult.getVal(i))); } return result; } private static void printHelp(Options options) { StringBuilder helpFooter = new StringBuilder(); helpFooter.append("Examples: \n"); helpFooter.append("snapshotscli.sh --create snapshot-1 -c books -z localhost:2181 \n"); helpFooter.append("snapshotscli.sh --list -c books -z localhost:2181 \n"); helpFooter.append("snapshotscli.sh --describe snapshot-1 -c books -z localhost:2181 \n"); helpFooter.append( "snapshotscli.sh --export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath -i req_0 \n"); helpFooter.append("snapshotscli.sh --delete snapshot-1 -c books -z localhost:2181 \n"); HelpFormatter formatter = new HelpFormatter(); formatter.setOptionComparator(new OptionComarator<>()); formatter.printHelp("SolrSnapshotsTool", null, options, helpFooter.toString(), false); } private static class OptionComarator<T extends Option> implements Comparator<T> { public int compare(T o1, T o2) { String s1 = o1.hasLongOpt() ? o1.getLongOpt() : o1.getOpt(); String s2 = o2.hasLongOpt() ? o2.getLongOpt() : o2.getOpt(); return OPTION_HELP_ORDER.indexOf(s1) - OPTION_HELP_ORDER.indexOf(s2); } } }