Java tutorial
/* * Autopsy Forensic Browser * * Copyright 2011-2016 Basis Technology Corp. * Contact: carrier <at> sleuthkit <dot> org * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.sleuthkit.autopsy.keywordsearch; import com.google.common.base.CharMatcher; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.commons.validator.routines.checkdigit.LuhnCheckDigit; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.TermsResponse.Term; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.datamodel.CreditCards; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** * Implements a regex query that will be performed as a two step operation. In * the first step, the Solr terms component is used to find any terms in the * index that match the regex. In the second step, term queries are executed for * each matched term to produce the set of keyword hits for the regex. */ final class TermsComponentQuery implements KeywordSearchQuery { private static final Logger LOGGER = Logger.getLogger(TermsComponentQuery.class.getName()); private static final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); private static final String SEARCH_HANDLER = "/terms"; //NON-NLS private static final String SEARCH_FIELD = Server.Schema.CONTENT_WS.toString(); private static final int TERMS_SEARCH_TIMEOUT = 90 * 1000; // Milliseconds private static final String CASE_INSENSITIVE = "case_insensitive"; //NON-NLS private static final boolean DEBUG_FLAG = Version.Type.DEVELOPMENT.equals(Version.getBuildType()); private static final int MAX_TERMS_QUERY_RESULTS = 20000; private final KeywordList keywordList; private final Keyword keyword; private String searchTerm; private boolean searchTermIsEscaped; private final List<KeywordQueryFilter> filters = new ArrayList<>(); // THIS APPEARS TO BE UNUSED /* * The following fields are part of the initial implementation of credit * card account search and should be factored into another class when time * permits. */ private static final Pattern CREDIT_CARD_NUM_PATTERN = Pattern.compile("(?<ccn>[3456]([ -]?\\d){11,18})"); //12-19 digits, with possible single spaces or dashes in between. First digit is 3,4,5, or 6 //NON-NLS private static final LuhnCheckDigit CREDIT_CARD_NUM_LUHN_CHECK = new LuhnCheckDigit(); private static final Pattern CREDIT_CARD_TRACK1_PATTERN = Pattern.compile( /* * Track 1 is alphanumeric. * * This regex matches 12-19 digit ccns embeded in a track 1 formated * string. This regex matches (and extracts groups) even if the * entire track is not present as long as the part that is conforms * to the track format. */ "(?:" //begin nested optinal group //NON-NLS + "%?" //optional start sentinal: % //NON-NLS + "B)?" //format code //NON-NLS + "(?<accountNumber>[3456]([ -]?\\d){11,18})" //12-19 digits, with possible single spaces or dashes in between. first digit is 3,4,5, or 6 //NON-NLS + "\\^" //separator //NON-NLS + "(?<name>[^^]{2,26})" //2-26 charachter name, not containing ^ //NON-NLS + "(?:\\^" //separator //NON-NLS + "(?:(?:\\^|(?<expiration>\\d{4}))" //separator or 4 digit expiration YYMM //NON-NLS + "(?:(?:\\^|(?<serviceCode>\\d{3}))"//separator or 3 digit service code //NON-NLS + "(?:(?<discretionary>[^?]*)" // discretionary data not containing separator //NON-NLS + "(?:\\?" // end sentinal: ? //NON-NLS + "(?<LRC>.)" //longitudinal redundancy check //NON-NLS + "?)?)?)?)?)?");//close nested optional groups //NON-NLS private static final Pattern CREDIT_CARD_TRACK2_PATTERN = Pattern.compile( /* * Track 2 is numeric plus six punctuation symbolls :;<=>? * * This regex matches 12-19 digit ccns embeded in a track 2 formated * string. This regex matches (and extracts groups) even if the * entire track is not present as long as the part that is conforms * to the track format. * */ "[:;<=>?]?" //(optional)start sentinel //NON-NLS + "(?<accountNumber>[3456]([ -]?\\d){11,18})" //12-19 digits, with possible single spaces or dashes in between. first digit is 3,4,5, or 6 //NON-NLS + "(?:[:;<=>?]" //separator //NON-NLS + "(?:(?<expiration>\\d{4})" //4 digit expiration date YYMM //NON-NLS + "(?:(?<serviceCode>\\d{3})" //3 digit service code //NON-NLS + "(?:(?<discretionary>[^:;<=>?]*)" //discretionary data, not containing punctuation marks //NON-NLS + "(?:[:;<=>?]" //end sentinel //NON-NLS + "(?<LRC>.)" //longitudinal redundancy check //NON-NLS + "?)?)?)?)?)?"); //close nested optional groups //NON-NLS private static final BlackboardAttribute.Type KEYWORD_SEARCH_DOCUMENT_ID = new BlackboardAttribute.Type( ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_DOCUMENT_ID); /** * Constructs an object that implements a regex query that will be performed * as a two step operation. In the first step, the Solr terms component is * used to find any terms in the index that match the regex. In the second * step, term queries are executed for each matched term to produce the set * of keyword hits for the regex. * * @param keywordList A keyword list that contains the keyword that provides * the regex search term for the query. * @param keyword The keyword that provides the regex search term for * the query. */ // TODO: Why is both the list and the keyword added to the state of this // object? // TODO: Why is the search term not escaped and given substring wildcards, // if needed, here in the constructor? TermsComponentQuery(KeywordList keywordList, Keyword keyword) { this.keywordList = keywordList; this.keyword = keyword; this.searchTerm = keyword.getSearchTerm(); } /** * Gets the keyword list that contains the keyword that provides the regex * search term for the query. * * @return The keyword list. */ @Override public KeywordList getKeywordList() { return keywordList; } /** * Gets the original search term for the query, without any escaping or, if * it is a literal term, the addition of wildcards for a substring search. * * @return The original search term. */ @Override public String getQueryString() { return keyword.getSearchTerm(); } /** * Indicates whether or not the search term for the query is a literal term * that needs have wildcards added to it to make the query a substring * search. * * @return True or false. */ @Override public boolean isLiteral() { return false; } /** * Adds wild cards to the search term for the query, which makes the query a * substring search, if it is a literal search term. */ @Override public void setSubstringQuery() { searchTerm = ".*" + searchTerm + ".*"; } /** * Escapes the search term for the query. */ @Override public void escape() { searchTerm = Pattern.quote(keyword.getSearchTerm()); searchTermIsEscaped = true; } /** * Indicates whether or not the search term has been escaped yet. * * @return True or false. */ @Override public boolean isEscaped() { return searchTermIsEscaped; } /** * Gets the escaped search term for the query, assuming it has been escaped * by a call to TermsComponentQuery.escape. * * @return The search term, possibly escaped. */ @Override public String getEscapedQueryString() { return this.searchTerm; } /** * Indicates whether or not the search term is a valid regex. * * @return True or false. */ @Override public boolean validate() { if (searchTerm.isEmpty()) { return false; } try { Pattern.compile(searchTerm); return true; } catch (IllegalArgumentException ex) { return false; } } /** * Does nothing, not applicable to a regex query, which always searches a * field created specifically for regex sesarches. * * @param field The name of a Solr document field to search. */ @Override public void setField(String field) { } /** * Adds a filter to the query. * * @param filter The filter. */ // TODO: Document this better. @Override public void addFilter(KeywordQueryFilter filter) { this.filters.add(filter); } /** * Executes the regex query as a two step operation. In the first step, the * Solr terms component is used to find any terms in the index that match * the regex. In the second step, term queries are executed for each matched * term to produce the set of keyword hits for the regex. * * @return A QueryResult object or null. * * @throws NoOpenCoreException */ @Override public QueryResults performQuery() throws KeywordSearchModuleException, NoOpenCoreException { /* * Do a query using the Solr terms component to find any terms in the * index that match the regex. */ final SolrQuery termsQuery = new SolrQuery(); termsQuery.setRequestHandler(SEARCH_HANDLER); termsQuery.setTerms(true); termsQuery.setTermsRegexFlag(CASE_INSENSITIVE); termsQuery.setTermsRegex(searchTerm); termsQuery.addTermsField(SEARCH_FIELD); termsQuery.setTimeAllowed(TERMS_SEARCH_TIMEOUT); termsQuery.setShowDebugInfo(DEBUG_FLAG); termsQuery.setTermsLimit(MAX_TERMS_QUERY_RESULTS); List<Term> terms = KeywordSearch.getServer().queryTerms(termsQuery).getTerms(SEARCH_FIELD); /* * Do a term query for each term that matched the regex. */ QueryResults results = new QueryResults(this, keywordList); for (Term term : terms) { /* * If searching for credit card account numbers, do a Luhn check on * the term and discard it if it does not pass. */ if (keyword.getArtifactAttributeType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { Matcher matcher = CREDIT_CARD_NUM_PATTERN.matcher(term.getTerm()); matcher.find(); final String ccn = CharMatcher.anyOf(" -").removeFrom(matcher.group("ccn")); if (false == CREDIT_CARD_NUM_LUHN_CHECK.isValid(ccn)) { continue; } } /* * Do an ordinary query with the escaped term and convert the query * results into a single list of keyword hits without duplicates. * * Note that the filters field appears to be unused. There is an old * comment here, what does it mean? "Note: we can't set filter query * on terms query but setting filter query on fileResults query will * yield the same result." The filter is NOT being added to the term * query. */ String escapedTerm = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); LuceneQuery termQuery = new LuceneQuery(keywordList, new Keyword(escapedTerm, true)); filters.forEach(termQuery::addFilter); // This appears to be unused QueryResults termQueryResult = termQuery.performQuery(); Set<KeywordHit> termHits = new HashSet<>(); for (Keyword word : termQueryResult.getKeywords()) { termHits.addAll(termQueryResult.getResults(word)); } results.addResult(new Keyword(term.getTerm(), false), new ArrayList<>(termHits)); } return results; } /** * Converts the keyword hits for a given search term into artifacts. * * @param searchTerm The search term. * @param hit The keyword hit. * @param snippet The document snippet that contains the hit * @param listName The name of the keyword list that contained the keyword * for which the hit was found. * * * * @return An object that wraps an artifact and a mapping by id of its * attributes. */ // TODO: Are we actually making meaningful use of the KeywordCachedArtifact // class? @Override public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String searchTerm, KeywordHit hit, String snippet, String listName) { /* * Create either a "plain vanilla" keyword hit artifact with keyword and * regex attributes, or a credit card account artifact with attributes * parsed from from the snippet for the hit and looked up based on the * parsed bank identifcation number. */ BlackboardArtifact newArtifact; Collection<BlackboardAttribute> attributes = new ArrayList<>(); if (keyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, searchTerm)); attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, keyword.getSearchTerm())); try { newArtifact = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS return null; } } else { /* * Parse the credit card account attributes from the snippet for the * hit. */ attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, MODULE_NAME, Account.Type.CREDIT_CARD.name())); Map<BlackboardAttribute.Type, BlackboardAttribute> parsedTrackAttributeMap = new HashMap<>(); Matcher matcher = CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet()); if (matcher.find()) { parseTrack1Data(parsedTrackAttributeMap, matcher); } matcher = CREDIT_CARD_TRACK2_PATTERN.matcher(hit.getSnippet()); if (matcher.find()) { parseTrack2Data(parsedTrackAttributeMap, matcher); } final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap .get(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)); if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) { if (hit.isArtifactHit()) { LOGGER.log(Level.SEVERE, String.format( "Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", searchTerm, hit.getSnippet(), hit.getArtifact().getArtifactID())); //NON-NLS } else { LOGGER.log(Level.SEVERE, String.format( "Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", searchTerm, hit.getSnippet(), hit.getContent().getId())); //NON-NLS } return null; } attributes.addAll(parsedTrackAttributeMap.values()); /* * Look up the bank name, scheme, etc. attributes for the bank * indentification number (BIN). */ final int bin = Integer.parseInt(ccnAttribute.getValueString().substring(0, 8)); CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin); if (binInfo != null) { binInfo.getScheme().ifPresent(scheme -> attributes .add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme))); binInfo.getCardType().ifPresent(cardType -> attributes .add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType))); binInfo.getBrand().ifPresent(brand -> attributes .add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand))); binInfo.getBankName().ifPresent(bankName -> attributes .add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName))); binInfo.getBankPhoneNumber().ifPresent(phoneNumber -> attributes .add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber))); binInfo.getBankURL().ifPresent( url -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url))); binInfo.getCountry().ifPresent(country -> attributes .add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country))); binInfo.getBankCity().ifPresent(city -> attributes .add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city))); } /* * If the hit is from unused or unallocated space, record the Solr * document id to support showing just the chunk that contained the * hit. */ if (hit.getContent() instanceof AbstractFile) { AbstractFile file = (AbstractFile) hit.getContent(); if (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) { attributes.add(new BlackboardAttribute(KEYWORD_SEARCH_DOCUMENT_ID, MODULE_NAME, hit.getSolrDocumentId())); } } /* * Create an account artifact. */ try { newArtifact = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_ACCOUNT); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding artifact for account to blackboard", ex); //NON-NLS return null; } } if (StringUtils.isNotBlank(listName)) { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); } if (snippet != null) { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); } if (hit.isArtifactHit()) { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID())); } try { newArtifact.addAttributes(attributes); KeywordCachedArtifact writeResult = new KeywordCachedArtifact(newArtifact); writeResult.add(attributes); return writeResult; } catch (TskCoreException e) { LOGGER.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS return null; } } /** * Parses the track 2 data from the snippet for a credit card account number * hit and turns them into artifact attributes. * * @param attributesMap A map of artifact attribute objects, used to avoid * creating duplicate attributes. * @param matcher A matcher for the snippet. */ static private void parseTrack2Data(Map<BlackboardAttribute.Type, BlackboardAttribute> attributesMap, Matcher matcher) { addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_NUMBER, "accountNumber", matcher); addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_EXPIRATION, "expiration", matcher); addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_SERVICE_CODE, "serviceCode", matcher); addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_DISCRETIONARY, "discretionary", matcher); addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_LRC, "LRC", matcher); } /** * Parses the track 1 data from the snippet for a credit card account number * hit and turns them into artifact attributes. The track 1 data has the * same fields as the track two data, plus the account holder's name. * * @param attributesMap A map of artifact attribute objects, used to avoid * creating duplicate attributes. * @param matcher A matcher for the snippet. */ static private void parseTrack1Data(Map<BlackboardAttribute.Type, BlackboardAttribute> attributeMap, Matcher matcher) { parseTrack2Data(attributeMap, matcher); addAttributeIfNotAlreadyCaptured(attributeMap, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher); } /** * Creates an attribute of the the given type to the given artifact with a * value parsed from the snippet for a credit account number hit. * * @param attributesMap A map of artifact attribute objects, used to avoid * creating duplicate attributes. * @param attrType The type of attribute to create. * @param groupName The group name of the regular expression that was * used to parse the attribute data. * @param matcher A matcher for the snippet. */ static private void addAttributeIfNotAlreadyCaptured( Map<BlackboardAttribute.Type, BlackboardAttribute> attributeMap, ATTRIBUTE_TYPE attrType, String groupName, Matcher matcher) { BlackboardAttribute.Type type = new BlackboardAttribute.Type(attrType); attributeMap.computeIfAbsent(type, (BlackboardAttribute.Type t) -> { String value = matcher.group(groupName); if (attrType.equals(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)) { value = CharMatcher.anyOf(" -").removeFrom(value); } if (StringUtils.isNotBlank(value)) { return new BlackboardAttribute(attrType, MODULE_NAME, value); } return null; }); } }