List of usage examples for org.springframework.batch.item ItemReader ItemReader
ItemReader
From source file:com.searchbox.collection.pubmed.PubmedCollection.java
public ItemReader<Resource> reader() { return new ItemReader<Resource>() { boolean hasmore = true; @Override/*from w w w .j a v a 2s .com*/ public Resource read() { if (hasmore) { hasmore = false; Resource resource = context.getResource("classpath:data/pubmedIndex.xml"); if (resource.exists()) { LOGGER.info("Read has created this resource: " + resource.getFilename()); return resource; } } return null; } }; }
From source file:com.searchbox.collection.oppfin.CordisCollection.java
public ItemReader<JSONObject> reader() throws IOException { return new ItemReader<JSONObject>() { JSONArray cordisData = new JSONArray(); Iterator iterator;/*from w w w . j a v a 2 s . com*/ { String url_fp7 = env.getProperty("data.fp7"); download(url_fp7); String url_h2020 = env.getProperty("data.h2020"); download(url_h2020); LOGGER.info("Starting collection / ItemReader"); String strFolder = context.getResource("classpath:data/xml").getFile().getAbsolutePath(); // String strFolder = "/tmp/xml"; LOGGER.info("=====>xml folder:" + strFolder); File folder = new File(strFolder); File[] fXml = folder.listFiles(); LOGGER.info("folder size:" + fXml.length); for (int i = 0; i < fXml.length; i++) { JSONObject json = readFromFile(fXml[i]); if (json == null) { continue; } else { cordisData.add(json); } } LOGGER.info("=====>Cordis data is parsed {}", cordisData.size()); iterator = cordisData.iterator(); } @Override public JSONObject read() { if (iterator.hasNext()) { return (JSONObject) iterator.next(); } return null; } }; }
From source file:com.searchbox.collection.oppfin.EENCollection.java
public ItemReader<Profile> reader() throws RemoteException { return new ItemReader<Profile>() { IPODServiceSOAPProxy eenService; ProfileQueryRequest request;/*from w w w .j av a 2s. c o m*/ List<Profile> profiles; Date date = new Date(System.currentTimeMillis()); Date start = new Date(System.currentTimeMillis()); DateFormat dfmt = new DateFormatManager("YYYYMMdd").getDateFormatInstance(); { // Get the service eenService = new IPODServiceSOAPProxy(); // Set the password request = new ProfileQueryRequest(); request.setUsername(env.getProperty(EEN_SERVICE_USER, EEN_SERVICE_USER_DEFAULT)); request.setPassword(env.getProperty(EEN_SERVICE_PWD, EEN_SERVICE_PWD_DEFAULT)); // Start with an empty list of profiles profiles = new ArrayList<Profile>(); date = start; } public void fillProfiles(Date from, Date to) { try { LOGGER.info("Fill Profile - ", from.toString(), to.toString()); // time to get some new profiles... JAXBElement<String> jaxbDeadlineDateAfter = new JAXBElement(new QName( "http://schemas.datacontract.org/2004/07/EEN.Merlin.Backend.Core.BO.PODService", "DeadlineDateAfter"), String.class, dfmt.format(from)); JAXBElement<String> jaxbDeadlineDateBefore = new JAXBElement(new QName( "http://schemas.datacontract.org/2004/07/EEN.Merlin.Backend.Core.BO.PODService", "DeadlineDateBefore"), String.class, dfmt.format(to)); request.setDeadlineDateAfter(jaxbDeadlineDateAfter); request.setDeadlineDateBefore(jaxbDeadlineDateBefore); Profile[] newProfiles = eenService.getProfiles(request); EENCollection.LOGGER.info("adding: " + newProfiles.length + " new profiles"); profiles.addAll(Arrays.asList(newProfiles)); } catch (Exception e) { LOGGER.error("Could not load profiles: " + e); } } public Profile read() throws RemoteException { if (start.after(DateUtils.addYears(date, 3))) { return null; } if (profiles.isEmpty()) { while (start.before(DateUtils.addYears(date, 1)) && profiles.isEmpty()) { Date end = DateUtils.addDays(start, 5); EENCollection.LOGGER.info("Fetching EEN profiles from deadline date " + dfmt.format(start) + " to deadline date" + dfmt.format(end)); fillProfiles(start, end); start = end; } if (start.after(DateUtils.addYears(date, 3))) { return null; } } if (profiles.size() == 0) { return null; } return profiles.remove(0); } }; }
From source file:com.searchbox.collection.oppfin.IdealISTCollection.java
public ItemReader<String> reader() { return new ItemReader<String>() { List<String> documents; int page = 0; {/*from ww w . ja va2 s. c o m*/ documents = new ArrayList<String>(); } @Override public String read() { if (documents.isEmpty()) { Document xmlDocuments = httpGet(RequestBuilder.get() .setUri(env.getProperty(IDEALIST_LIST_SERVICE, IDEALIST_LIST_SERVICE_DEFAULT)) .addParameter("pageNum", Integer.toString(page)).addParameter("pageSize", "10")); NodeList documentList = xmlDocuments.getElementsByTagName("document"); for (int i = 0; i < documentList.getLength(); i++) { String uid = documentList.item(i).getAttributes().getNamedItem("uid").getNodeValue(); documents.add(uid); } page++; } if (documents.isEmpty()) { return null; } else { return documents.remove(0); } } }; }
From source file:com.searchbox.collection.oppfin.TopicCollection.java
public ItemReader<JSONObject> topicReader() { return new ItemReader<JSONObject>() { String topicList = loadFromUrl(env.getProperty(TOPIC_LIST_URL, TOPIC_LIST_URL_DEFAULT)); // Topic list is a json list and we are interested in topicData => // Topics => all Object obj = JSONValue.parse(topicList); JSONObject jsonObject = (JSONObject) obj; JSONObject topicData = (JSONObject) jsonObject.get("topicData"); JSONArray topics = (JSONArray) topicData.get("Topics"); // loop array Iterator<JSONObject> iterator = topics.iterator(); public JSONObject read() { if (iterator.hasNext()) { return iterator.next(); }/*from w w w . j a v a 2 s . c o m*/ return null; } }; }
From source file:com.searchbox.collection.oppfin.TopicCollection.java
public ItemReader<JSONObject> callReader() { return new ItemReader<JSONObject>() { String callListJSON = loadFromUrl(env.getProperty(CALL_LIST_URL, CALL_LIST_URL_DEFAULT)); // Topic list is a json list and we are interested in topicData => // Topics => all Object obj = JSONValue.parse(callListJSON); JSONObject jsonObject = (JSONObject) obj; JSONObject callData = (JSONObject) jsonObject.get("callData"); JSONArray calls = (JSONArray) callData.get("Calls"); // loop array Iterator<JSONObject> iterator = calls.iterator(); public JSONObject read() { if (iterator.hasNext()) { return iterator.next(); }//from ww w .j a v a 2 s.c om return null; } }; }
From source file:org.springframework.batch.core.step.item.FaultTolerantStepFactoryBeanRetryTests.java
@Test public void testCacheLimitWithRetry() throws Exception { factory.setRetryLimit(2);//w w w . java 2 s .c o m factory.setCommitInterval(3); // sufficiently high so we never hit it factory.setSkipLimit(10); // set the cache limit stupidly low factory.setRetryContextCache(new MapRetryContextCache(0)); ItemReader<String> provider = new ItemReader<String>() { @Override public String read() { String item = "" + count; provided.add(item); count++; if (count >= 10) { // prevent infinite loop in worst case scenario return null; } return item; } }; ItemWriter<String> itemWriter = new ItemWriter<String>() { @Override public void write(List<? extends String> item) throws Exception { processed.addAll(item); logger.debug("Write Called! Item: [" + item + "]"); throw new RuntimeException("Write error - planned but retryable."); } }; factory.setItemReader(provider); factory.setItemWriter(itemWriter); AbstractStep step = (AbstractStep) factory.getObject(); StepExecution stepExecution = new StepExecution(step.getName(), jobExecution); repository.add(stepExecution); step.execute(stepExecution); assertEquals(BatchStatus.FAILED, stepExecution.getStatus()); // We added a bogus cache so no items are actually skipped // because they aren't recognised as eligible assertEquals(0, stepExecution.getSkipCount()); // [0, 1, 2] assertEquals(3, provided.size()); // [0, 1, 2] assertEquals(3, processed.size()); // [] assertEquals(0, recovered.size()); }