Example usage for com.amazonaws.services.s3.model ObjectListing getMarker

List of usage examples for com.amazonaws.services.s3.model ObjectListing getMarker

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. ObjectListing
  6. getMarker

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getMarker.

Prototype

public String getMarker()

Source Link

Document

Gets the marker parameter originally used to request this object listing, or null if no marker was specified.

Usage

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception {
    ListBucketResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;
    try {//from ww w  . ja v a 2s.  c  o m
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(request.getBucket());
        listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter());
        listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker());
        listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys())));
        listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix());

        ObjectListing response = s3Client.listObjects(listRequest);

        /* Non-optional, must have non-null values */
        reply.setName(request.getBucket());
        reply.setMaxKeys(response.getMaxKeys());
        reply.setMarker(response.getMarker() == null ? "" : response.getMarker());
        reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix());
        reply.setIsTruncated(response.isTruncated());

        /* Optional */
        reply.setNextMarker(response.getNextMarker());
        reply.setDelimiter(response.getDelimiter());
        if (reply.getContents() == null) {
            reply.setContents(new ArrayList<ListEntry>());
        }
        if (reply.getCommonPrefixesList() == null) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());
        }

        for (S3ObjectSummary obj : response.getObjectSummaries()) {
            //Add entry, note that the canonical user is set based on requesting user, not returned user
            reply.getContents()
                    .add(new ListEntry(obj.getKey(),
                            DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(),
                            obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass()));
        }

        if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());

            for (String s : response.getCommonPrefixes()) {
                reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s));
            }
        }

        return reply;
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }
}

From source file:com.upplication.s3fs.util.AmazonS3ClientMock.java

License:Open Source License

@Override
public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) {

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(previousObjectListing.getBucketName());
    objectListing.setPrefix(previousObjectListing.getPrefix());
    objectListing.setMarker(previousObjectListing.getMarker());
    objectListing.setDelimiter(previousObjectListing.getDelimiter());

    if (!previousObjectListing.isTruncated() || previousObjectListing.getNextMarker() == null) {
        return objectListing;
    }/*from  w  w  w  . j  a v a2s. co m*/

    Bucket bucket = find(previousObjectListing.getBucketName());
    Iterator<S3Element> iterator = objects.get(bucket).iterator();

    int i = 0;
    boolean continueElement = false;

    while (iterator.hasNext()) {

        S3Element elem = iterator.next();

        if (!continueElement && elem.getS3Object().getKey().equals(previousObjectListing.getNextMarker())) {
            continueElement = true;
        }

        if (continueElement) {
            // TODO. add delimiter and marker support
            if (previousObjectListing.getPrefix() != null
                    && elem.getS3Object().getKey().startsWith(previousObjectListing.getPrefix())) {

                S3ObjectSummary s3ObjectSummary = parseToS3ObjectSummary(elem);
                objectListing.getObjectSummaries().add(s3ObjectSummary);
                // max 1000 elements at same time.
                if (i + 1 == LIMIT_AWS_MAX_ELEMENTS && iterator.hasNext()) {
                    objectListing.setTruncated(true);
                    objectListing.setNextMarker(iterator.next().getS3Object().getKey());
                    return objectListing;
                } else {
                    objectListing.setTruncated(false);
                }

                i++;
            }
        }
    }

    return objectListing;
}

From source file:jp.sanix.analysis.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    if (args.length < 1) {
        System.out.println("Please specify at least an arguments that is pvs_serial_id.");
        System.out.println("analysis A9990004 // get today's data");
        System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
        System.out.println("analysis A9990004 2015/05/30 2015/06/30");
        System.exit(-1);//from  ww w  .  j a v a  2s  .  com
    }

    String id = args[0];

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);

    System.out.println(id);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    while (cal.before(end)) {
        xlsSheet xls = new xlsSheet(json);
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        //         String bucketName = "pvdata-storage-staging";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        json = line.substring(line.indexOf("{"));
                        recent = parseDate(line.substring(0, 25));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + id + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3
            .listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis").withPrefix(FOLDER));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + "index.html", file));
}

From source file:jp.sanix.analysisLocal.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    BufferedReader br = new BufferedReader(
            new InputStreamReader(new FileInputStream(new File("canceled.csv")), "SJIS"));
    br.readLine();/*  ww  w.  j a  va2s  .c om*/

    String l;
    while ((l = br.readLine()) != null) {
        String[] col = l.split(",");
        String id = col[0];
        String datefrom = col[1];
        String dateto = col[1];

        SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
        format.setTimeZone(TimeZone.getTimeZone("JST"));
        SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

        Calendar cal = Calendar.getInstance();
        Calendar end = Calendar.getInstance();
        String today = toDate(cal);

        try {
            cal.setTime(format.parse(datefrom));
        } catch (ParseException e) {
        }
        try {
            end.setTime(format.parse(dateto));
            end.add(Calendar.DAY_OF_MONTH, 1);
        } catch (ParseException e) {
        }

        AmazonS3 s3 = new AmazonS3Client();
        s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

        Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery(
                "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        db.close();
        Date recent = new Date();

        while (cal.before(end)) {
            xlsSheet xls = new xlsSheet(json);
            System.out.println("Getting data of " + toDate(cal));

            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            recent = parseDate(line.substring(0, 25));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);

            /* if today, read postgres to get recent data */
            if (toDate(cal).equals(today)) {
                System.out.println("Get recent data from postgres");
                try {
                    db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                    st = db.createStatement();
                    String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                            + pgformat.format(recent) + "';";
                    System.out.println(sql);
                    rs = st.executeQuery(sql);
                    while (rs.next()) {
                        json = rs.getString(1);
                        xls.putData(json);
                    }
                    rs.close();
                    db.close();
                } catch (PSQLException e) {
                } catch (ParseException e) {
                }
            }
            System.out.println("Write Buffer");
            xls.writeBuffer();

            File file = new File(
                    "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "_" + toDate(cal).replace("/", "-") + ".xlsx");
            xls.putFile(new FileOutputStream(file));
            System.out.println("Finished: " + toDate(cal));

            cal.add(Calendar.DAY_OF_MONTH, 1);
        }
    }
    br.close();
}

From source file:jp.sanix.analysisStaging.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    //      if (args.length < 1) {
    //         System.out.println("Please specify at least an arguments that is pvs_serial_id.");
    //         System.out.println("analysis A9990004 // get today's data");
    //         System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
    //         System.out.println("analysis A9990004 2015/05/30 2015/06/30");
    //         System.exit(-1);
    //      }// w  w  w. j  a va2 s  . co  m

    //      String id = args[0];
    String uniq = "763ff280b16c61f524ae31b33401bcd6";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    cal.setTime(format.parse("2015/11/23"));
    end.setTime(format.parse("2015/11/23"));

    System.out.println(uniq);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String bucketName = "pvdata-storage-staging";

    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        //         String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + uniq + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + uniq + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        xlsSheet xls = null;
        do {
            Boolean first = true;
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        String json = line.substring(line.indexOf("{"));
                        if (first) {
                            xls = new xlsSheet(json);
                            first = false;
                        } else {
                            xls.putData(json);
                        }
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + uniq + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

}

From source file:jp.sanix.mukaino.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String[] ids = { "A0000089", "A0002570" };

    Connection db;// w ww .  j ava  2 s.c o  m
    if (System.getProperty("user.name").toString().equals("ec2-user")) {
        db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    } else {
        db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
    }

    for (String id : ids) {
        pvSensor.getInstance().getById(id);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id
                + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        st.close();

        Calendar cal = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1??

        Calendar end = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1???

        do {
            String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1,
                    cal.get(Calendar.DAY_OF_MONTH));
            xlsSheetMukaino xls = new xlsSheetMukaino(json);
            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key,
                    cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)));
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + mDate.replace("-", "/")));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                    System.out.print(".");
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);
            System.out.println();

            xls.writeBuffer();
            File file = File.createTempFile("temp", ".xlsx");
            file.deleteOnExit();
            xls.putFile(new FileOutputStream(file));
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file));
            System.out.println("Finished: " + mDate);
            cal.add(Calendar.DAY_OF_MONTH, 1);
        } while (cal.before(end));

    }
    db.close();

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis")
            .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/"));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(
            new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file));
}

From source file:jp.sanix.Rawdata.java

License:Open Source License

public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException {

    TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>();
    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
    fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
    Calendar cal = Calendar.getInstance();
    cal.setTime(this.getDate());

    /* AWS S3????? */
    String bucketName = "pvdata-storage-production";
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/"));

    /* get data from s3 */
    do {/*from w w  w .j  a  va2  s  . co m*/
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname));
            BufferedReader reader = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
            while (reader.ready()) {
                String line = reader.readLine();
                int pos;
                try {
                    pos = line.indexOf("{");
                } catch (NullPointerException e) {
                    continue;
                }
                try {
                    String jsons = line.substring(pos);
                    JSONObject json = new JSONObject(jsons);
                    JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                    for (int i = 0; i < arr.length(); i++) {
                        JSONObject obj = arr.getJSONObject(i);
                        Date date = fmt.parse(obj.getString("time"));
                        for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                            if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                                Double genkwh;
                                genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                                JSONObject jobj;
                                if (data.containsKey(date)) {
                                    jobj = data.get(date);
                                } else {
                                    jobj = new JSONObject();
                                }
                                if (genkwh != null) {
                                    jobj.put(String.valueOf(j), genkwh);
                                }
                                if (jobj.length() > 0) {
                                    data.put(date, jobj);
                                }
                            }
                        }
                    }
                } catch (JSONException e) {
                    e.printStackTrace();
                } catch (StringIndexOutOfBoundsException e) {
                    e.printStackTrace();
                }
            }
            reader.close();
            object.close();
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);

    Calendar today = Calendar.getInstance();
    today.setTimeZone(TimeZone.getTimeZone("JST"));
    if (toDate(cal).equals(toDate(today))) {
        SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgfmt.setTimeZone(TimeZone.getTimeZone("UTC"));
        Calendar recent = Calendar.getInstance();
        recent.setTimeZone(TimeZone.getTimeZone("UST"));
        recent.add(Calendar.HOUR, -2);
        Connection db;
        if (System.getProperty("user.name").toString().equals("ec2-user")) {
            db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        } else {
            db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
        }
        Statement st = db.createStatement();
        String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode()
                + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';";
        ResultSet rs = st.executeQuery(sql);
        while (rs.next()) {
            String jsons = rs.getString(1);
            try {
                JSONObject json = new JSONObject(jsons);
                JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                for (int i = 0; i < arr.length(); i++) {
                    JSONObject obj = arr.getJSONObject(i);
                    Date date = fmt.parse(obj.getString("time"));
                    for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                        if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                            Double genkwh;
                            genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                            JSONObject jobj;
                            if (data.containsKey(date)) {
                                jobj = data.get(date);
                            } else {
                                jobj = new JSONObject();
                            }
                            if (genkwh != null) {
                                jobj.put(String.valueOf(j), genkwh);
                            }
                            if (jobj.length() > 0) {
                                data.put(date, jobj);
                            }
                        }
                    }
                }
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        rs.close();
        db.close();
    }
    return data;
}

From source file:jp.sanix.weatherData.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    BufferedReader br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));

    String line = null;//w ww .  j a v  a2 s. co m
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_insolation/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_insolation.csv", file));
        }
    }
    br.close();

    br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_temperature/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_temperture.csv", file));
        }
    }
    br.close();
}

From source file:jp.sanix.yokusei.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    String id = "A0002441";
    String datefrom = "2015/10/01";
    String dateto = "2015/10/12";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);//from  w w  w. jav  a  2 s  . c  om

    try {
        cal.setTime(format.parse(datefrom));
    } catch (ParseException e) {
    }
    try {
        end.setTime(format.parse(dateto));
        end.add(Calendar.DAY_OF_MONTH, 1);
    } catch (ParseException e) {
    }

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    xlsSheetYokusei xls = new xlsSheetYokusei(json);
    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                String line;
                while ((line = reader.readLine()) != null) {
                    try {
                        json = line.substring(line.indexOf("{"));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }
    File file = new File(
            "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "-Diamond" + toDate(cal).replace("/", "-") + ".xlsx");
    xls.putFile(new FileOutputStream(file));
    System.out.println("Finished: " + toDate(cal));
}