Example usage for com.amazonaws.services.s3.model GetObjectRequest GetObjectRequest

List of usage examples for com.amazonaws.services.s3.model GetObjectRequest GetObjectRequest

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. GetObjectRequest
  6. GetObjectRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model GetObjectRequest GetObjectRequest.

Prototype

public GetObjectRequest(String bucketName, String key)

Source Link

Document

Constructs a new GetObjectRequest with all the required parameters.

Usage

From source file:io.minio.awssdk.tests.S3TestUtils.java

License:Apache License

void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, String expectedMD5, int start,
        int length) throws Exception, IOException {
    GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName).withSSECustomerKey(sseKey);

    if (start >= 0 && length >= 0) {
        getObjectRequest.setRange(start, start + length - 1);
    }/*from  ww  w. ja v  a 2  s  .  c  o m*/

    S3Object s3Object = s3Client.getObject(getObjectRequest);

    int size = 0;
    int c;

    S3ObjectInputStream input = s3Object.getObjectContent();

    ByteArrayOutputStream output = new ByteArrayOutputStream();
    String data = "";
    while ((c = input.read()) != -1) {
        output.write((byte) c);
        size++;
    }

    if (length >= 0 && size != length) {
        throw new Exception(
                "downloaded object has unexpected size, expected: " + length + ", received: " + size);
    }

    String calculatedMD5 = Utils.getBufferMD5(output.toByteArray());

    if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) {
        throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: "
                + calculatedMD5);

    }
}

From source file:io.seldon.resources.external.S3FileStreamer.java

License:Apache License

public InputStream getResourceStream(String reference) throws IOException {
    logger.info("Reading file from s3://" + reference);
    AmazonS3Client client;/*from   ww w.  j ava  2  s. co  m*/
    if (creds != null) {
        client = new AmazonS3Client(creds);
    } else {
        client = new AmazonS3Client();
    }
    String[] bucketAndFile = reference.split("/", 2);
    if (bucketAndFile.length != 2) {
        return null;
    }
    S3Object object = client.getObject(new GetObjectRequest(bucketAndFile[0], bucketAndFile[1]));
    if (reference.endsWith(".gz")) {
        return new S3ObjectInputStreamWrapper(new GZIPInputStream(object.getObjectContent()), client);

    } else {
        return new S3ObjectInputStreamWrapper(object.getObjectContent(), client);
    }
}

From source file:jp.inc.forrest.aws.swf.AverageCalculatorActivitiesImpl.java

License:Open Source License

@Override
public int computeSumForChunk(String bucketName, String filename, int chunkNumber, int chunkSize)
        throws IOException {
    int sum = 0;//from   w  w w . j  a  v a 2  s .c  om
    int from = chunkNumber * chunkSize;
    int to = from + chunkSize;

    int offset = chunkNumber * chunkSize * ROW_SIZE;
    int bytesToRead = chunkSize * ROW_SIZE;

    // Create a request to download content for computing the sum for this
    // chunk
    GetObjectRequest getRequest = new GetObjectRequest(bucketName, filename);
    getRequest.setRange(offset, offset + bytesToRead - 1);

    // Download content
    S3Object obj = storage.getObject(getRequest);
    InputStream inputStream = obj.getObjectContent();
    InputStreamReader inputStreamReader = null;
    BufferedReader reader = null;
    try {
        String line = null;
        inputStreamReader = new InputStreamReader(inputStream);
        reader = new BufferedReader(inputStreamReader);

        // Compute sum for downloaded content
        while ((line = reader.readLine()) != null) {
            sum += Integer.parseInt(line);
        }
    } finally {
        if (reader != null)
            reader.close();
        if (inputStreamReader != null)
            inputStreamReader.close();
        if (inputStream != null)
            inputStream.close();
    }

    System.out.printf("Sum from '%d' to '%d' is: '%d'\n", from + 1, to, sum);
    return sum;
}

From source file:jp.sanix.analysis.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    if (args.length < 1) {
        System.out.println("Please specify at least an arguments that is pvs_serial_id.");
        System.out.println("analysis A9990004 // get today's data");
        System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
        System.out.println("analysis A9990004 2015/05/30 2015/06/30");
        System.exit(-1);//w  w  w  . j av a 2 s  .  c o  m
    }

    String id = args[0];

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);

    System.out.println(id);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    while (cal.before(end)) {
        xlsSheet xls = new xlsSheet(json);
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        //         String bucketName = "pvdata-storage-staging";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        json = line.substring(line.indexOf("{"));
                        recent = parseDate(line.substring(0, 25));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + id + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3
            .listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis").withPrefix(FOLDER));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + "index.html", file));
}

From source file:jp.sanix.analysisLocal.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    BufferedReader br = new BufferedReader(
            new InputStreamReader(new FileInputStream(new File("canceled.csv")), "SJIS"));
    br.readLine();// w ww .ja va  2  s. com

    String l;
    while ((l = br.readLine()) != null) {
        String[] col = l.split(",");
        String id = col[0];
        String datefrom = col[1];
        String dateto = col[1];

        SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
        format.setTimeZone(TimeZone.getTimeZone("JST"));
        SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

        Calendar cal = Calendar.getInstance();
        Calendar end = Calendar.getInstance();
        String today = toDate(cal);

        try {
            cal.setTime(format.parse(datefrom));
        } catch (ParseException e) {
        }
        try {
            end.setTime(format.parse(dateto));
            end.add(Calendar.DAY_OF_MONTH, 1);
        } catch (ParseException e) {
        }

        AmazonS3 s3 = new AmazonS3Client();
        s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

        Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery(
                "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        db.close();
        Date recent = new Date();

        while (cal.before(end)) {
            xlsSheet xls = new xlsSheet(json);
            System.out.println("Getting data of " + toDate(cal));

            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            recent = parseDate(line.substring(0, 25));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);

            /* if today, read postgres to get recent data */
            if (toDate(cal).equals(today)) {
                System.out.println("Get recent data from postgres");
                try {
                    db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                    st = db.createStatement();
                    String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                            + pgformat.format(recent) + "';";
                    System.out.println(sql);
                    rs = st.executeQuery(sql);
                    while (rs.next()) {
                        json = rs.getString(1);
                        xls.putData(json);
                    }
                    rs.close();
                    db.close();
                } catch (PSQLException e) {
                } catch (ParseException e) {
                }
            }
            System.out.println("Write Buffer");
            xls.writeBuffer();

            File file = new File(
                    "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "_" + toDate(cal).replace("/", "-") + ".xlsx");
            xls.putFile(new FileOutputStream(file));
            System.out.println("Finished: " + toDate(cal));

            cal.add(Calendar.DAY_OF_MONTH, 1);
        }
    }
    br.close();
}

From source file:jp.sanix.analysisStaging.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    //      if (args.length < 1) {
    //         System.out.println("Please specify at least an arguments that is pvs_serial_id.");
    //         System.out.println("analysis A9990004 // get today's data");
    //         System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
    //         System.out.println("analysis A9990004 2015/05/30 2015/06/30");
    //         System.exit(-1);
    //      }//from  w  w  w  .ja  va 2  s .co m

    //      String id = args[0];
    String uniq = "763ff280b16c61f524ae31b33401bcd6";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    cal.setTime(format.parse("2015/11/23"));
    end.setTime(format.parse("2015/11/23"));

    System.out.println(uniq);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String bucketName = "pvdata-storage-staging";

    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        //         String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + uniq + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + uniq + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        xlsSheet xls = null;
        do {
            Boolean first = true;
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        String json = line.substring(line.indexOf("{"));
                        if (first) {
                            xls = new xlsSheet(json);
                            first = false;
                        } else {
                            xls.putData(json);
                        }
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + uniq + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

}

From source file:jp.sanix.mukaino.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String[] ids = { "A0000089", "A0002570" };

    Connection db;/*from   w w w.j a  v a 2 s  .  c  o m*/
    if (System.getProperty("user.name").toString().equals("ec2-user")) {
        db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    } else {
        db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
    }

    for (String id : ids) {
        pvSensor.getInstance().getById(id);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id
                + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        st.close();

        Calendar cal = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1??

        Calendar end = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1???

        do {
            String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1,
                    cal.get(Calendar.DAY_OF_MONTH));
            xlsSheetMukaino xls = new xlsSheetMukaino(json);
            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key,
                    cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)));
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + mDate.replace("-", "/")));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                    System.out.print(".");
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);
            System.out.println();

            xls.writeBuffer();
            File file = File.createTempFile("temp", ".xlsx");
            file.deleteOnExit();
            xls.putFile(new FileOutputStream(file));
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file));
            System.out.println("Finished: " + mDate);
            cal.add(Calendar.DAY_OF_MONTH, 1);
        } while (cal.before(end));

    }
    db.close();

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis")
            .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/"));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(
            new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file));
}

From source file:jp.sanix.Rawdata.java

License:Open Source License

public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException {

    TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>();
    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
    fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
    Calendar cal = Calendar.getInstance();
    cal.setTime(this.getDate());

    /* AWS S3????? */
    String bucketName = "pvdata-storage-production";
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/"));

    /* get data from s3 */
    do {//  w ww.j  a v a 2s .c o m
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname));
            BufferedReader reader = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
            while (reader.ready()) {
                String line = reader.readLine();
                int pos;
                try {
                    pos = line.indexOf("{");
                } catch (NullPointerException e) {
                    continue;
                }
                try {
                    String jsons = line.substring(pos);
                    JSONObject json = new JSONObject(jsons);
                    JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                    for (int i = 0; i < arr.length(); i++) {
                        JSONObject obj = arr.getJSONObject(i);
                        Date date = fmt.parse(obj.getString("time"));
                        for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                            if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                                Double genkwh;
                                genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                                JSONObject jobj;
                                if (data.containsKey(date)) {
                                    jobj = data.get(date);
                                } else {
                                    jobj = new JSONObject();
                                }
                                if (genkwh != null) {
                                    jobj.put(String.valueOf(j), genkwh);
                                }
                                if (jobj.length() > 0) {
                                    data.put(date, jobj);
                                }
                            }
                        }
                    }
                } catch (JSONException e) {
                    e.printStackTrace();
                } catch (StringIndexOutOfBoundsException e) {
                    e.printStackTrace();
                }
            }
            reader.close();
            object.close();
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);

    Calendar today = Calendar.getInstance();
    today.setTimeZone(TimeZone.getTimeZone("JST"));
    if (toDate(cal).equals(toDate(today))) {
        SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgfmt.setTimeZone(TimeZone.getTimeZone("UTC"));
        Calendar recent = Calendar.getInstance();
        recent.setTimeZone(TimeZone.getTimeZone("UST"));
        recent.add(Calendar.HOUR, -2);
        Connection db;
        if (System.getProperty("user.name").toString().equals("ec2-user")) {
            db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        } else {
            db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
        }
        Statement st = db.createStatement();
        String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode()
                + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';";
        ResultSet rs = st.executeQuery(sql);
        while (rs.next()) {
            String jsons = rs.getString(1);
            try {
                JSONObject json = new JSONObject(jsons);
                JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                for (int i = 0; i < arr.length(); i++) {
                    JSONObject obj = arr.getJSONObject(i);
                    Date date = fmt.parse(obj.getString("time"));
                    for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                        if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                            Double genkwh;
                            genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                            JSONObject jobj;
                            if (data.containsKey(date)) {
                                jobj = data.get(date);
                            } else {
                                jobj = new JSONObject();
                            }
                            if (genkwh != null) {
                                jobj.put(String.valueOf(j), genkwh);
                            }
                            if (jobj.length() > 0) {
                                data.put(date, jobj);
                            }
                        }
                    }
                }
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        rs.close();
        db.close();
    }
    return data;
}

From source file:jp.sanix.weatherData.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    BufferedReader br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));

    String line = null;/*from w  ww. ja  va 2 s.co  m*/
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_insolation/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_insolation.csv", file));
        }
    }
    br.close();

    br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_temperature/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_temperture.csv", file));
        }
    }
    br.close();
}

From source file:jp.sanix.yokusei.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    String id = "A0002441";
    String datefrom = "2015/10/01";
    String dateto = "2015/10/12";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);/*  w w  w .  ja v a 2  s .  co m*/

    try {
        cal.setTime(format.parse(datefrom));
    } catch (ParseException e) {
    }
    try {
        end.setTime(format.parse(dateto));
        end.add(Calendar.DAY_OF_MONTH, 1);
    } catch (ParseException e) {
    }

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    xlsSheetYokusei xls = new xlsSheetYokusei(json);
    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                String line;
                while ((line = reader.readLine()) != null) {
                    try {
                        json = line.substring(line.indexOf("{"));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }
    File file = new File(
            "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "-Diamond" + toDate(cal).replace("/", "-") + ".xlsx");
    xls.putFile(new FileOutputStream(file));
    System.out.println("Finished: " + toDate(cal));
}