Example usage for com.amazonaws.services.cloudsearchdomain.model UploadDocumentsRequest UploadDocumentsRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.cloudsearchdomain.model UploadDocumentsRequest UploadDocumentsRequest.

Prototype

UploadDocumentsRequest

Source Link

Usage

From source file:com.clicktravel.infrastructure.persistence.aws.cloudsearch.CloudSearchEngine.java

License:Apache License

private UploadDocumentsRequest uploadDocumentsRequest(
        final BatchDocumentUpdateRequest batchDocumentUpdateRequest) {
    final UploadDocumentsRequest uploadDocumentsRequest = new UploadDocumentsRequest();
    final byte[] documentUpdatesJsonBytes;
    final String documentUpdatesJson = JsonDocumentUpdateMarshaller
            .marshall(batchDocumentUpdateRequest.getDocumentUpdates());
    documentUpdatesJsonBytes = documentUpdatesJson.getBytes(Charset.forName("UTF-8"));
    final InputStream documents = new ByteArrayInputStream(documentUpdatesJsonBytes);
    uploadDocumentsRequest.setDocuments(documents);
    uploadDocumentsRequest.setContentLength((long) documentUpdatesJsonBytes.length);
    uploadDocumentsRequest.setContentType(MediaType.APPLICATION_JSON);
    return uploadDocumentsRequest;
}

---

From source file:com.digitalpebble.stormcrawler.aws.bolt.CloudSearchIndexerBolt.java

License:Apache License

public void sendBatch() {

    timeLastBatchSent = System.currentTimeMillis();

    // nothing to do
    if (numDocsInBatch == 0) {
        return;//from   w  w  w . j  a  v  a2s .  c o  m
    }

    // close the array
    buffer.append(']');

    LOG.info("Sending {} docs to CloudSearch", numDocsInBatch);

    byte[] bb = buffer.toString().getBytes(StandardCharsets.UTF_8);

    if (dumpBatchFilesToTemp) {
        try {
            File temp = File.createTempFile("CloudSearch_", ".json");
            FileUtils.writeByteArrayToFile(temp, bb);
            LOG.info("Wrote batch file {}", temp.getName());
            // ack the tuples
            for (Tuple t : unacked) {
                _collector.ack(t);
            }
            unacked.clear();
        } catch (IOException e1) {
            LOG.error("Exception while generating batch file", e1);
            // fail the tuples
            for (Tuple t : unacked) {
                _collector.fail(t);
            }
            unacked.clear();
        } finally {
            // reset buffer and doc counter
            buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
            numDocsInBatch = 0;
        }
        return;
    }
    // not in debug mode
    try (InputStream inputStream = new ByteArrayInputStream(bb)) {
        UploadDocumentsRequest batch = new UploadDocumentsRequest();
        batch.setContentLength((long) bb.length);
        batch.setContentType(ContentType.Applicationjson);
        batch.setDocuments(inputStream);
        UploadDocumentsResult result = client.uploadDocuments(batch);
        LOG.info(result.getStatus());
        for (DocumentServiceWarning warning : result.getWarnings()) {
            LOG.info(warning.getMessage());
        }
        if (!result.getWarnings().isEmpty()) {
            eventCounter.scope("Warnings").incrBy(result.getWarnings().size());
        }
        eventCounter.scope("Added").incrBy(result.getAdds());
        // ack the tuples
        for (Tuple t : unacked) {
            _collector.ack(t);
        }
        unacked.clear();
    } catch (Exception e) {
        LOG.error("Exception while sending batch", e);
        LOG.error(buffer.toString());
        // fail the tuples
        for (Tuple t : unacked) {
            _collector.fail(t);
        }
        unacked.clear();
    } finally {
        // reset buffer and doc counter
        buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
        numDocsInBatch = 0;
    }
}

---

From source file:com.easarrive.aws.plugins.common.service.impl.CloudSearchService.java

License:Open Source License

/**
 * {@inheritDoc}/*from   w  w w . j  a va2  s  . c o  m*/
 */
@Override
public UploadDocumentsResult uploadDocumentsByJson(AmazonCloudSearchDomain domainClient, String endpoint,
        InputStream documents, Long contentLength) {
    domainClient.setEndpoint(endpoint);

    UploadDocumentsRequest uploadDocumentsRequest = new UploadDocumentsRequest();
    uploadDocumentsRequest.withDocuments(documents).withContentType(ContentType.Applicationjson)
            .withContentLength(contentLength);

    UploadDocumentsResult result = domainClient.uploadDocuments(uploadDocumentsRequest);
    return result;
}

---

From source file:com.easarrive.aws.plugins.common.service.impl.CloudSearchService.java

License:Open Source License

/**
 * {@inheritDoc}/* w w  w. j a  v a2  s.c  o m*/
 */
@Override
public UploadDocumentsResult uploadDocumentsByXML(AmazonCloudSearchDomain domainClient, String domainName,
        InputStream documents, Long contentLength) {
    domainClient.setEndpoint(domainName);

    UploadDocumentsRequest uploadDocumentsRequest = new UploadDocumentsRequest();
    uploadDocumentsRequest.withDocuments(documents).withContentType(ContentType.Applicationxml)
            .withContentLength(contentLength);

    UploadDocumentsResult result = domainClient.uploadDocuments(uploadDocumentsRequest);
    return result;
}

---

From source file:com.norconex.committer.cloudsearch.CloudSearchCommitter.java

License:Apache License

private void uploadBatchToCloudSearch(List<JSONObject> documentBatch) {
    // Convert the JSON list to String and read it as a stream from memory
    // (for increased performance), for it to be usable by the AWS 
    // CloudSearch UploadRequest. If memory becomes a concern, consider 
    // streaming to file.
    // ArrayList.toString() joins the elements in a JSON-compliant way.
    byte[] bytes;
    try {//from  w ww . j a  v a  2  s  .c  o m
        bytes = documentBatch.toString().getBytes(CharEncoding.UTF_8);
    } catch (UnsupportedEncodingException e) {
        throw new CommitterException("UTF-8 not supported by OS.", e);
    }
    try (ByteArrayInputStream is = new ByteArrayInputStream(bytes)) {
        UploadDocumentsRequest uploadRequest = new UploadDocumentsRequest();
        uploadRequest.setContentType("application/json");
        uploadRequest.setDocuments(is);
        uploadRequest.setContentLength((long) bytes.length);
        ensureAWSClient();
        UploadDocumentsResult result = awsClient.uploadDocuments(uploadRequest);
        LOG.info(result.getAdds() + " Add requests and " + result.getDeletes() + " Delete requests "
                + "sent to the AWS CloudSearch domain.");
    } catch (IOException e) {
        throw new CommitterException("Could not upload request to CloudSearch.", e);
    }
}

---

From source file:org.apache.nutch.indexwriter.cloudsearch.CloudSearchIndexWriter.java

License:Apache License

@Override
public void commit() throws IOException {

    // nothing to do
    if (numDocsInBatch == 0) {
        return;/*  w w w .j  ava  2 s  .  com*/
    }

    // close the array
    buffer.append(']');

    LOG.info("Sending {} docs to CloudSearch", numDocsInBatch);

    byte[] bb = buffer.toString().getBytes(StandardCharsets.UTF_8);

    if (dumpBatchFilesToTemp) {
        try {
            File temp = File.createTempFile("CloudSearch_", ".json");
            FileUtils.writeByteArrayToFile(temp, bb);
            LOG.info("Wrote batch file {}", temp.getName());
        } catch (IOException e1) {
            LOG.error("Exception while generating batch file", e1);
        } finally {
            // reset buffer and doc counter
            buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
            numDocsInBatch = 0;
        }
        return;
    }
    // not in debug mode
    try (InputStream inputStream = new ByteArrayInputStream(bb)) {
        UploadDocumentsRequest batch = new UploadDocumentsRequest();
        batch.setContentLength((long) bb.length);
        batch.setContentType(ContentType.Applicationjson);
        batch.setDocuments(inputStream);
        @SuppressWarnings("unused")
        UploadDocumentsResult result = client.uploadDocuments(batch);
    } catch (Exception e) {
        LOG.error("Exception while sending batch", e);
        LOG.error(buffer.toString());
    } finally {
        // reset buffer and doc counter
        buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
        numDocsInBatch = 0;
    }
}

---