Example usage for com.amazonaws.services.cloudsearchdomain.model ContentType Applicationjson

Introduction

In this page you can find the example usage for com.amazonaws.services.cloudsearchdomain.model ContentType Applicationjson.

Prototype

ContentType Applicationjson

To view the source code for com.amazonaws.services.cloudsearchdomain.model ContentType Applicationjson.

Click Source Link

Usage

From source file:com.digitalpebble.stormcrawler.aws.bolt.CloudSearchIndexerBolt.java

License:Apache License

public void sendBatch() {

    timeLastBatchSent = System.currentTimeMillis();

    // nothing to do
    if (numDocsInBatch == 0) {
        return;/*from   ww w .ja v  a2s  .  com*/
    }

    // close the array
    buffer.append(']');

    LOG.info("Sending {} docs to CloudSearch", numDocsInBatch);

    byte[] bb = buffer.toString().getBytes(StandardCharsets.UTF_8);

    if (dumpBatchFilesToTemp) {
        try {
            File temp = File.createTempFile("CloudSearch_", ".json");
            FileUtils.writeByteArrayToFile(temp, bb);
            LOG.info("Wrote batch file {}", temp.getName());
            // ack the tuples
            for (Tuple t : unacked) {
                _collector.ack(t);
            }
            unacked.clear();
        } catch (IOException e1) {
            LOG.error("Exception while generating batch file", e1);
            // fail the tuples
            for (Tuple t : unacked) {
                _collector.fail(t);
            }
            unacked.clear();
        } finally {
            // reset buffer and doc counter
            buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
            numDocsInBatch = 0;
        }
        return;
    }
    // not in debug mode
    try (InputStream inputStream = new ByteArrayInputStream(bb)) {
        UploadDocumentsRequest batch = new UploadDocumentsRequest();
        batch.setContentLength((long) bb.length);
        batch.setContentType(ContentType.Applicationjson);
        batch.setDocuments(inputStream);
        UploadDocumentsResult result = client.uploadDocuments(batch);
        LOG.info(result.getStatus());
        for (DocumentServiceWarning warning : result.getWarnings()) {
            LOG.info(warning.getMessage());
        }
        if (!result.getWarnings().isEmpty()) {
            eventCounter.scope("Warnings").incrBy(result.getWarnings().size());
        }
        eventCounter.scope("Added").incrBy(result.getAdds());
        // ack the tuples
        for (Tuple t : unacked) {
            _collector.ack(t);
        }
        unacked.clear();
    } catch (Exception e) {
        LOG.error("Exception while sending batch", e);
        LOG.error(buffer.toString());
        // fail the tuples
        for (Tuple t : unacked) {
            _collector.fail(t);
        }
        unacked.clear();
    } finally {
        // reset buffer and doc counter
        buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
        numDocsInBatch = 0;
    }
}

---

From source file:com.easarrive.aws.plugins.common.service.impl.CloudSearchService.java

License:Open Source License

/**
 * {@inheritDoc}/*from w w w .j a v  a2  s .  c om*/
 */
@Override
public UploadDocumentsResult uploadDocumentsByJson(AmazonCloudSearchDomain domainClient, String endpoint,
        InputStream documents, Long contentLength) {
    domainClient.setEndpoint(endpoint);

    UploadDocumentsRequest uploadDocumentsRequest = new UploadDocumentsRequest();
    uploadDocumentsRequest.withDocuments(documents).withContentType(ContentType.Applicationjson)
            .withContentLength(contentLength);

    UploadDocumentsResult result = domainClient.uploadDocuments(uploadDocumentsRequest);
    return result;
}

---

From source file:org.apache.nutch.indexwriter.cloudsearch.CloudSearchIndexWriter.java

License:Apache License

@Override
public void commit() throws IOException {

    // nothing to do
    if (numDocsInBatch == 0) {
        return;//  w  ww . j ava2  s  .  c o  m
    }

    // close the array
    buffer.append(']');

    LOG.info("Sending {} docs to CloudSearch", numDocsInBatch);

    byte[] bb = buffer.toString().getBytes(StandardCharsets.UTF_8);

    if (dumpBatchFilesToTemp) {
        try {
            File temp = File.createTempFile("CloudSearch_", ".json");
            FileUtils.writeByteArrayToFile(temp, bb);
            LOG.info("Wrote batch file {}", temp.getName());
        } catch (IOException e1) {
            LOG.error("Exception while generating batch file", e1);
        } finally {
            // reset buffer and doc counter
            buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
            numDocsInBatch = 0;
        }
        return;
    }
    // not in debug mode
    try (InputStream inputStream = new ByteArrayInputStream(bb)) {
        UploadDocumentsRequest batch = new UploadDocumentsRequest();
        batch.setContentLength((long) bb.length);
        batch.setContentType(ContentType.Applicationjson);
        batch.setDocuments(inputStream);
        @SuppressWarnings("unused")
        UploadDocumentsResult result = client.uploadDocuments(batch);
    } catch (Exception e) {
        LOG.error("Exception while sending batch", e);
        LOG.error(buffer.toString());
    } finally {
        // reset buffer and doc counter
        buffer = new StringBuffer(MAX_SIZE_BATCH_BYTES).append('[');
        numDocsInBatch = 0;
    }
}

---