List of usage examples for org.jsoup.nodes Document getElementsByAttributeValue
public Elements getElementsByAttributeValue(String key, String value)
From source file:tkbautobooking.BookingSystem.java
private void praseLoginPage() throws Exception { Document doc = Jsoup.parse(LoginPageHTML); Elements token_elm = doc.getElementsByAttributeValue("name", "access_token"); if (token_elm.isEmpty() || token_elm.size() > 1 || !token_elm.first().hasAttr("value")) throw new Exception("Prase Login Page fail !"); login_hidden_token = token_elm.first().attr("value"); }
From source file:ac.simons.oembed.Oembed.java
private OembedProvider autodiscoverOembedURIForUrl(final String url) { OembedProvider rv = null;/*from w ww . j ava 2s.c om*/ try { final HttpGet request = new HttpGet(url); final HttpResponse httpResponse = this.httpClient.execute(request); if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) logger.warn(String.format("Autodiscovery for %s failed, server returned error %d: %s", url, httpResponse.getStatusLine().getStatusCode(), EntityUtils.toString(httpResponse.getEntity()))); else { final URI uri = request.getURI(); final Document document = Jsoup.parse(EntityUtils.toString(httpResponse.getEntity(), "UTF-8"), String.format("%s://%s:%d", uri.getScheme(), uri.getHost(), uri.getPort())); for (Element alternate : document.getElementsByAttributeValue("rel", "alternate")) { if (alternate.attr("type").equalsIgnoreCase("application/json+oembed")) rv = new AutodiscoveredOembedProvider(url, new URI(alternate.absUrl("href")), "json"); else if (alternate.attr("type").equalsIgnoreCase("text/xml+oembed")) rv = new AutodiscoveredOembedProvider(url, new URI(alternate.absUrl("href")), "xml"); if (rv != null) break; } } } catch (Exception e) { logger.warn(String.format("Autodiscovery for %s failedd: %s", url, e.getMessage()), e); } return rv; }
From source file:mx.itdurango.rober.siitdocentes.ActivityAlumnos.java
/** * Permite descomponer el cdigo html que se enva con una estructura especifica para llenar los datos de la vista * * @param html cdigo html que se recibi de una peticin HttpGet, debe tener una estructura similar a la siguiente para que el proceso funcione correctamente * <p/>/* w w w. j a v a 2 s . c om*/ * <input name="periodo" type="hidden" value="20141" /> * <input name="materia" type="hidden" value="SD2424" /> * <input name="grupo" type="hidden" value="5VR" /> * <input name="docente" type="hidden" value="LOQR841213822" /> * <input name="fecha_captura" type="hidden" value="2014/06/12" /> * <table> * <tr> * <td>No</td> * <td>Noctrl</td> * <td>Nombre</td> * <td>Unidad 1</td> * <td>Unidad 1</td> * <td>Unidad 3</td> * <td>...</td> * <td>Unidad N</td> * </tr> * <tr> * <td>1</td> * <td>9999999</td> * <td>XXXXXXXXXXXXXXXXXXXXX</td> * <td><input type="text" name="calif[1][1]" value="999"/></td> * <td><input type="text" name="calif[1][2]" value="999"/></td> * <td><input type="text" name="calif[1][3]" value="999"/></td> * <td>...</td> * <td><input type="text" name="calif[1][N]" value="999"/></td> * </tr> * <tr> * <td>2</td> * <td>888888888</td> * <td>YYYYYYYYYYYYYYYYYYYYY</td> * <td><input type="text" name="calif[2][1]" value="999"/></td> * <td><input type="text" name="calif[2][2]" value="999"/></td> * <td><input type="text" name="calif[2][3]" value="999"/></td> * <td>...</td> * <td><input type="text" name="calif[2][N]" value="999"/></td> * </tr> * <tr> * <td>M</td> * <td>000000000</td> * <td>ZZZZZZZZZZZZZZZZZZZZZZ</td> * <td><input type="text" name="calif[M][1]" value="999"/></td> * <td><input type="text" name="calif[M][2]" value="999"/></td> * <td><input type="text" name="calif[M][3]" value="999"/></td> * <td>...</td> * <td><input type="text" name="calif[M][N]" value="999"/></td> * </tr> * </table> */ void llenaAlumnos(String html) { //Generar un archivo de documento para almacenar los datos del html de forma que se pueda //manipular facilmente usando la librera Jsoup Document doc = Jsoup.parse(html); try { //extraer los valores de los elementos del formulario y almacenarlos en los atributos correspondientes de la clase Elements e = doc.getElementsByAttributeValue("name", "periodo"); periodo = e.get(0).attr("value"); e = doc.getElementsByAttributeValue("name", "materia"); materia = e.get(0).attr("value"); e = doc.getElementsByAttributeValue("name", "grupo"); grupo = e.get(0).attr("value"); e = doc.getElementsByAttributeValue("name", "docente"); docente = e.get(0).attr("value"); e = doc.getElementsByAttributeValue("name", "fecha_captura"); fecha_captura = e.get(0).attr("value"); //extraer la tabla correspondiente al listado de alumnos en el caso del siit.itdurango.edu.mx, // corresponde a la tabla numero 2 y ya que la numeracin comienza en 0, la tabla que necesitamos est en el indice 1 Element tabla = doc.getElementsByTag("table").get(1); //Extraer todos los elementos de tipo tr que pertenecen a la tabla y almacenarlos en una coleccion de tipo Elements. Elements renglones = tabla.getElementsByTag("tr"); //Recorrer la coleccin de renglones y almacenar cada uno en un objeto for (Element tr : renglones) { //para cada objeto tr, extraer sus elementos td y almacenarlos en una coleccion Elements tds = tr.getElementsByTag("td"); //permite llevar el control de la columna que se est leyendo, ya que las columnas no tienen un id o clase, se realiza el proceso a mano. int col = 1; //contenedor de tipo AlumosParciales para almacenar la informacin de cada alumno (tr) AlumnosParciales c = new AlumnosParciales(); for (Element td : tds) { if (col == 1) {// la columna 1 corresponde al nmero consecutivo de la tabla c.setNum(td.html()); } else if (col == 2) {// la columna 2 corresponde al nmero de control del alumno c.setControl(td.html()); } else if (col == 3) {// la columna 3 corresponde al nombre del alumno c.setNombre(Estaticos.sanitize(td.html())); } else { //el resto de las columnas pertenecen a las calificaciones parciales //se extrae el elemento <input> de la columna y se obtiene el atributo valor para recuperar la calificacin en caso de que ya hubiera sido asignada String cal = td.getElementsByTag("input").get(0).attr("value"); ArrayList<String> calif = c.getCalificaciones(); calif.add(cal); //se agrega la nueva calificacin al conjunto de calificaciones del alumno c.setCalificaciones(calif); } col++; //incrementa el numero de columa } if (c.getCalificaciones().size() > 0) { //para evitar agregar al listado de alumnos el encabezado de la tabla, validamos que existan calificaciones. gcs.add(c); } } //Llenamos el spinner de unidades a partir del numero de calificaciones que existen en el arreglo List<String> spinnerArray = new ArrayList<String>(); for (int i = 1; i <= gcs.get(1).getCalificaciones().size() - 1; i++) { spinnerArray.add("Unidad " + i); } ArrayAdapter<String> adapter = new ArrayAdapter<String>(this, android.R.layout.simple_spinner_item, spinnerArray); adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); spn_unidad.setAdapter(adapter); //llenamos el listado de alumnos con la informacin que se obtuvo del proceso anterior alumnosParcialesAdapter = new AlumnosParcialesAdapter(this, gcs, unidad); lvAlumnos.setAdapter(alumnosParcialesAdapter); } catch (Exception e) { e.printStackTrace(); Toast.makeText(this, getString(R.string.error_parser), Toast.LENGTH_SHORT).show(); finish(); //finaliza el intent actual para desplegar el anterior } }
From source file:com.geecko.QuickLyric.tasks.IdDecoder.java
@Override protected Lyrics doInBackground(String... strings) { String url = strings[0];/* w ww . j a v a 2 s . c o m*/ String artist; String track; if (url.contains("//www.soundhound.com/")) { try { // todo switch to Jsoup String html = getUrlAsString(url); int preceding = html.indexOf("root.App.trackDa") + 19; int following = html.substring(preceding).indexOf(";"); String data = html.substring(preceding, preceding + following); JSONObject jsonData = new JSONObject(data); artist = jsonData.getString("artist_display_name"); track = jsonData.getString("track_name"); } catch (IOException | JSONException e) { e.printStackTrace(); return new Lyrics(ERROR); } } else if (url.contains("//shz.am/")) { try { Document doc = Jsoup.connect(url.trim()).get(); track = doc.getElementsByAttribute("data-track-title").text(); artist = doc.getElementsByAttribute("data-track-artist").text(); } catch (IOException e) { e.printStackTrace(); return new Lyrics(ERROR); } } else if (url.contains("//play.google.com/store/music/")) { String docID = url.substring(url.indexOf("&tid=") + 5); try { Document doc = Jsoup.connect(url).get(); Element playCell = doc.getElementsByAttributeValue("data-track-docid", docID).get(0); artist = doc.getElementsByClass("primary").text(); track = playCell.parent().parent().child(1).getElementsByClass("title").text(); } catch (IOException e) { e.printStackTrace(); return new Lyrics(ERROR); } } else return new Lyrics(ERROR); Lyrics res = new Lyrics(Lyrics.SEARCH_ITEM); res.setArtist(artist); res.setTitle(track); return res; }
From source file:net.kevxu.purdueassist.course.CatalogDetail.java
private CatalogDetailEntry parseDocument(Document document) throws HtmlParseException, CourseNotFoundException, IOException { CatalogDetailEntry entry = new CatalogDetailEntry(subject, cnbr); Elements tableElements = document.getElementsByAttributeValue("summary", "This table lists the course detail for the selected term."); if (tableElements.isEmpty() != true) { // get name try {//from w w w . j a v a2 s . co m Element body = tableElements.first().select("tbody").first(); String nameBlock = body.select("tr td.nttitle").first().text(); String[] temp = nameBlock.split(subject.name() + " " + String.valueOf(cnbr)); String name = temp[temp.length - 1].substring(3); entry.setName(name); // get description body = body.select(".ntdefault").first(); String text = body.text(); int split = text.indexOf("Levels:"); String description = text.substring(0, split); description = description.substring(20); entry.setDescription(description); // get levels int begin = split; int end = text.indexOf("Schedule Types:"); String levels = text.substring(begin + 8, end); temp = levels.split("[ ,]"); List<String> lvs = new ArrayList<String>(); for (String s : temp) if (!s.equals("")) { lvs.add(s); } entry.setLevels(lvs); // get type and prerequisites List<Type> types = new ArrayList<Type>(); List<String> preq = new ArrayList<String>(); Elements parsing_A = body.select("a"); for (Element e : parsing_A) { if (e.attr("href").contains("schd_in") && !(e.attr("href").contains("%"))) { try { types.add(Type.valueOf(e.text().replace(" ", ""))); } catch (Exception exception) { throw new HtmlParseException(); } } else if (e.attr("href").contains("sel_attr=")) { preq.add(e.text()); } } if (types.size() > 0) entry.setType(types); if (preq.size() > 0) entry.setPrerequisites(preq); // get offered by begin = text.indexOf("Offered By:"); end = text.indexOf("Department:"); if (end < 0) end = text.indexOf("Course Attributes:"); if (end > 0) { entry.setOfferedBy(text.substring(begin + 12, end - 1)); } // get department begin = text.indexOf("Department:"); if (begin > 0) { end = text.indexOf("Course Attributes:"); entry.setDepartment((text.substring(begin + 12, end - 1))); } // get campus begin = text.indexOf("May be offered at any of the following campuses:"); String campuses; end = text.indexOf("Repeatable for Additional Credit:"); if (end < 0) end = text.indexOf("Learning Objectives:"); if (end < 0) end = text.indexOf("Restrictions:"); if (end < 0) end = text.indexOf("Corequisites:"); if (end < 0) end = text.indexOf("Prerequisites:"); if (end < 0) { campuses = text .substring(begin + "May be offered at any of the following campuses:".length() + 5); } else { campuses = text.substring( begin + "May be offered at any of the following campuses:".length() + 5, end - 1); } temp = campuses.replace(" ", "#").split("#"); List<String> camps = new ArrayList<String>(); for (String s : temp) { if (s.length() > 1) { camps.add(s); } } entry.setCampuses(camps); // get restrictions begin = text.indexOf("Restrictions:"); end = text.indexOf("Corequisites:"); if (end < 0) end = text.indexOf("Prerequisites:"); if (begin > 0 && end < 0) { entry.setRestrictions( text.substring(begin + "Restrictions:".length()).replace(" ", "\n")); } else if (begin > 0) { entry.setRestrictions( text.substring(begin + "Restrictions:".length(), end).replace(" ", "\n")); } } catch (StringIndexOutOfBoundsException e) { // no type, not available // System.out.println("-----------"); // System.out.println("Error for cnbr = " + cnbr); // System.out.println("-----------"); } } else { throw new CourseNotFoundException(); } return entry; }
From source file:biz.shadowservices.DegreesToolbox.DataFetcher.java
public FetchResult updateData(Context context, boolean force) { //Open database DBOpenHelper dbhelper = new DBOpenHelper(context); SQLiteDatabase db = dbhelper.getWritableDatabase(); // check for internet connectivity try {//w w w . j a v a2s .co m if (!isOnline(context)) { Log.d(TAG, "We do not seem to be online. Skipping Update."); return FetchResult.NOTONLINE; } } catch (Exception e) { exceptionReporter.reportException(Thread.currentThread(), e, "Exception during isOnline()"); } SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(context); if (!force) { try { if (sp.getBoolean("loginFailed", false) == true) { Log.d(TAG, "Previous login failed. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Previous login failed. Skipping Update."); return FetchResult.LOGINFAILED; } if (sp.getBoolean("autoupdates", true) == false) { Log.d(TAG, "Automatic updates not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Automatic updates not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } if (!isBackgroundDataEnabled(context) && sp.getBoolean("obeyBackgroundData", true)) { Log.d(TAG, "Background data not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Background data not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } if (!isAutoSyncEnabled() && sp.getBoolean("obeyAutoSync", true) && sp.getBoolean("obeyBackgroundData", true)) { Log.d(TAG, "Auto sync not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Auto sync not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } if (isWifi(context) && !sp.getBoolean("wifiUpdates", true)) { Log.d(TAG, "On wifi, and wifi auto updates not allowed. Skipping Update"); DBLog.insertMessage(context, "i", TAG, "On wifi, and wifi auto updates not allowed. Skipping Update"); return FetchResult.NOTALLOWED; } else if (!isWifi(context)) { Log.d(TAG, "We are not on wifi."); if (!isRoaming(context) && !sp.getBoolean("2DData", true)) { Log.d(TAG, "Automatic updates on 2Degrees data not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Automatic updates on 2Degrees data not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } else if (isRoaming(context) && !sp.getBoolean("roamingData", false)) { Log.d(TAG, "Automatic updates on roaming mobile data not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Automatic updates on roaming mobile data not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } } } catch (Exception e) { exceptionReporter.reportException(Thread.currentThread(), e, "Exception while finding if to update."); } } else { Log.d(TAG, "Update Forced"); } try { String username = sp.getString("username", null); String password = sp.getString("password", null); if (username == null || password == null) { DBLog.insertMessage(context, "i", TAG, "Username or password not set."); return FetchResult.USERNAMEPASSWORDNOTSET; } // Find the URL of the page to send login data to. Log.d(TAG, "Finding Action. "); HttpGetter loginPageGet = new HttpGetter("https://secure.2degreesmobile.co.nz/web/ip/login"); String loginPageString = loginPageGet.execute(); if (loginPageString != null) { Document loginPage = Jsoup.parse(loginPageString, "https://secure.2degreesmobile.co.nz/web/ip/login"); Element loginForm = loginPage.getElementsByAttributeValue("name", "loginFrm").first(); String loginAction = loginForm.attr("action"); // Send login form List<NameValuePair> loginValues = new ArrayList<NameValuePair>(); loginValues.add(new BasicNameValuePair("externalURLRedirect", "")); loginValues.add(new BasicNameValuePair("hdnAction", "login_userlogin")); loginValues.add(new BasicNameValuePair("hdnAuthenticationType", "M")); loginValues.add(new BasicNameValuePair("hdnlocale", "")); loginValues.add(new BasicNameValuePair("userid", username)); loginValues.add(new BasicNameValuePair("password", password)); Log.d(TAG, "Sending Login "); HttpPoster sendLoginPoster = new HttpPoster(loginAction, loginValues); // Parse result String loginResponse = sendLoginPoster.execute(); Document loginResponseParsed = Jsoup.parse(loginResponse); // Determine if this is a pre-pay or post-paid account. boolean postPaid; if (loginResponseParsed .getElementById("p_CustomerPortalPostPaidHomePage_WAR_customerportalhomepage") == null) { Log.d(TAG, "Pre-pay account or no account."); postPaid = false; } else { Log.d(TAG, "Post-paid account."); postPaid = true; } String homepageUrl = "https://secure.2degreesmobile.co.nz/group/ip/home"; if (postPaid) { homepageUrl = "https://secure.2degreesmobile.co.nz/group/ip/postpaid"; } HttpGetter homepageGetter = new HttpGetter(homepageUrl); String homepageHTML = homepageGetter.execute(); Document homePage = Jsoup.parse(homepageHTML); Element accountSummary = homePage.getElementById("accountSummary"); if (accountSummary == null) { Log.d(TAG, "Login failed."); return FetchResult.LOGINFAILED; } db.delete("cache", "", null); /* This code fetched some extra details for postpaid users, but on reflection they aren't that useful. * Might reconsider this. * if (postPaid) { Element accountBalanceSummaryTable = accountSummary.getElementsByClass("tableBillSummary").first(); Elements rows = accountBalanceSummaryTable.getElementsByTag("tr"); int rowno = 0; for (Element row : rows) { if (rowno > 1) { break; } //Log.d(TAG, "Starting row"); //Log.d(TAG, row.html()); Double value; try { Element amount = row.getElementsByClass("tableBillamount").first(); String amountHTML = amount.html(); Log.d(TAG, amountHTML.substring(1)); value = Double.parseDouble(amountHTML.substring(1)); } catch (Exception e) { Log.d(TAG, "Failed to parse amount from row."); value = null; } String expiresDetails = ""; String expiresDate = null; String name = null; try { Element details = row.getElementsByClass("tableBilldetail").first(); name = details.ownText(); Element expires = details.getElementsByTag("em").first(); if (expires != null) { expiresDetails = expires.text(); } Log.d(TAG, expiresDetails); Pattern pattern; pattern = Pattern.compile("\\(payment is due (.*)\\)"); Matcher matcher = pattern.matcher(expiresDetails); if (matcher.find()) { /*Log.d(TAG, "matched expires"); Log.d(TAG, "group 0:" + matcher.group(0)); Log.d(TAG, "group 1:" + matcher.group(1)); Log.d(TAG, "group 2:" + matcher.group(2)); * String expiresDateString = matcher.group(1); Date expiresDateObj; if (expiresDateString != null) { if (expiresDateString.length() > 0) { try { expiresDateObj = DateFormatters.EXPIRESDATE.parse(expiresDateString); expiresDate = DateFormatters.ISO8601DATEONLYFORMAT.format(expiresDateObj); } catch (java.text.ParseException e) { Log.d(TAG, "Could not parse date: " + expiresDateString); } } } } } catch (Exception e) { Log.d(TAG, "Failed to parse details from row."); } String expirev = null; ContentValues values = new ContentValues(); values.put("name", name); values.put("value", value); values.put("units", "$NZ"); values.put("expires_value", expirev ); values.put("expires_date", expiresDate); db.insert("cache", "value", values ); rowno++; } } */ Element accountSummaryTable = accountSummary.getElementsByClass("tableAccountSummary").first(); Elements rows = accountSummaryTable.getElementsByTag("tr"); for (Element row : rows) { // We are now looking at each of the rows in the data table. //Log.d(TAG, "Starting row"); //Log.d(TAG, row.html()); Double value; String units; try { Element amount = row.getElementsByClass("tableBillamount").first(); String amountHTML = amount.html(); //Log.d(TAG, amountHTML); String[] amountParts = amountHTML.split(" ", 2); //Log.d(TAG, amountParts[0]); //Log.d(TAG, amountParts[1]); if (amountParts[0].contains("Included") || amountParts[0].equals("All You Need") || amountParts[0].equals("Unlimited Text*")) { value = Values.INCLUDED; } else { try { value = Double.parseDouble(amountParts[0]); } catch (NumberFormatException e) { exceptionReporter.reportException(Thread.currentThread(), e, "Decoding value."); value = 0.0; } } units = amountParts[1]; } catch (NullPointerException e) { //Log.d(TAG, "Failed to parse amount from row."); value = null; units = null; } Element details = row.getElementsByClass("tableBilldetail").first(); String name = details.getElementsByTag("strong").first().text(); Element expires = details.getElementsByTag("em").first(); String expiresDetails = ""; if (expires != null) { expiresDetails = expires.text(); } Log.d(TAG, expiresDetails); Pattern pattern; if (postPaid == false) { pattern = Pattern.compile("\\(([\\d\\.]*) ?\\w*? ?expiring on (.*)\\)"); } else { pattern = Pattern.compile("\\(([\\d\\.]*) ?\\w*? ?will expire on (.*)\\)"); } Matcher matcher = pattern.matcher(expiresDetails); Double expiresValue = null; String expiresDate = null; if (matcher.find()) { /*Log.d(TAG, "matched expires"); Log.d(TAG, "group 0:" + matcher.group(0)); Log.d(TAG, "group 1:" + matcher.group(1)); Log.d(TAG, "group 2:" + matcher.group(2)); */ try { expiresValue = Double.parseDouble(matcher.group(1)); } catch (NumberFormatException e) { expiresValue = null; } String expiresDateString = matcher.group(2); Date expiresDateObj; if (expiresDateString != null) { if (expiresDateString.length() > 0) { try { expiresDateObj = DateFormatters.EXPIRESDATE.parse(expiresDateString); expiresDate = DateFormatters.ISO8601DATEONLYFORMAT.format(expiresDateObj); } catch (java.text.ParseException e) { Log.d(TAG, "Could not parse date: " + expiresDateString); } } } } ContentValues values = new ContentValues(); values.put("name", name); values.put("value", value); values.put("units", units); values.put("expires_value", expiresValue); values.put("expires_date", expiresDate); db.insert("cache", "value", values); } if (postPaid == false) { Log.d(TAG, "Getting Value packs..."); // Find value packs HttpGetter valuePacksPageGet = new HttpGetter( "https://secure.2degreesmobile.co.nz/group/ip/prevaluepack"); String valuePacksPageString = valuePacksPageGet.execute(); //DBLog.insertMessage(context, "d", "", valuePacksPageString); if (valuePacksPageString != null) { Document valuePacksPage = Jsoup.parse(valuePacksPageString); Elements enabledPacks = valuePacksPage.getElementsByClass("yellow"); for (Element enabledPack : enabledPacks) { Element offerNameElemt = enabledPack .getElementsByAttributeValueStarting("name", "offername").first(); if (offerNameElemt != null) { String offerName = offerNameElemt.val(); DBLog.insertMessage(context, "d", "", "Got element: " + offerName); ValuePack[] packs = Values.valuePacks.get(offerName); if (packs == null) { DBLog.insertMessage(context, "d", "", "Offer name: " + offerName + " not matched."); } else { for (ValuePack pack : packs) { ContentValues values = new ContentValues(); values.put("plan_startamount", pack.value); values.put("plan_name", offerName); DBLog.insertMessage(context, "d", "", "Pack " + pack.type.id + " start value set to " + pack.value); db.update("cache", values, "name = '" + pack.type.id + "'", null); } } } } } } SharedPreferences.Editor prefedit = sp.edit(); Date now = new Date(); prefedit.putString("updateDate", DateFormatters.ISO8601FORMAT.format(now)); prefedit.putBoolean("loginFailed", false); prefedit.putBoolean("networkError", false); prefedit.commit(); DBLog.insertMessage(context, "i", TAG, "Update Successful"); return FetchResult.SUCCESS; } } catch (ClientProtocolException e) { DBLog.insertMessage(context, "w", TAG, "Network error: " + e.getMessage()); return FetchResult.NETWORKERROR; } catch (IOException e) { DBLog.insertMessage(context, "w", TAG, "Network error: " + e.getMessage()); return FetchResult.NETWORKERROR; } finally { db.close(); } return null; }
From source file:net.kevxu.purdueassist.course.ScheduleDetail.java
private ScheduleDetailEntry parseDocument(Document document) throws HtmlParseException, CourseNotFoundException, ResultNotMatchException { ScheduleDetailEntry entry = new ScheduleDetailEntry(term, crn); Elements tableElements = document.getElementsByAttributeValue("summary", "This table is used to present the detailed class information."); if (!tableElements.isEmpty()) { for (Element tableElement : tableElements) { // get basic info for selected course Element tableBasicInfoElement = tableElement.getElementsByClass("ddlabel").first(); if (tableBasicInfoElement != null) { setBasicInfo(entry, tableBasicInfoElement.text()); } else { throw new HtmlParseException("Basic info element empty."); }/* www .ja va 2s.c o m*/ // get detailed course info Element tableDetailedInfoElement = tableElement.getElementsByClass("dddefault").first(); if (tableDetailedInfoElement != null) { // process seat info Elements tableSeatDetailElements = tableDetailedInfoElement.getElementsByAttributeValue( "summary", "This layout table is used to present the seating numbers."); if (tableSeatDetailElements.size() == 1) { Element tableSeatDetailElement = tableSeatDetailElements.first(); Elements tableSeatDetailEntryElements = tableSeatDetailElement.getElementsByTag("tbody") .first().children(); if (tableSeatDetailEntryElements.size() == 3 || tableSeatDetailEntryElements.size() == 4) { setSeats(entry, tableSeatDetailEntryElements.get(1).text()); setWaitlistSeats(entry, tableSeatDetailEntryElements.get(2).text()); if (tableSeatDetailEntryElements.size() == 4) { setCrosslistSeats(entry, tableSeatDetailEntryElements.get(3).text()); } } else { throw new HtmlParseException("Seat detail entry elements size not 3. We have " + tableSeatDetailEntryElements.size() + "."); } } else { throw new HtmlParseException( "Seat detail elements size not 1. We have " + tableSeatDetailElements.size() + "."); } // remove the seat info from detailed info tableSeatDetailElements.remove(); // remaining information setRemainingInfo(entry, tableDetailedInfoElement.html()); } else { throw new HtmlParseException("Detailed info element empty."); } } } else { // test empty Elements informationElements = document.getElementsByAttributeValue("summary", "This layout table holds message information"); if (!informationElements.isEmpty() && informationElements.text().contains("No detailed class information found")) { throw new CourseNotFoundException(informationElements.text()); } else { throw new HtmlParseException( "Course table not found, but page does not contain message stating no course found."); } } return entry; }
From source file:com.salsaberries.narchiver.Trawler.java
/** * Logs into the site./* ww w . j a v a2 s . c om*/ * * @return * @throws TrawlException */ private boolean login() throws TrawlException { --loginAttempts; if (loginAttempts < 0) { logger.error("Warning! Exceeded maximum number of login attempts! Program is now exiting."); throw new TrawlException("Maximum login attempts exceeded."); } logger.info("Attempting to log in at " + baseURL + site.getString("LOGIN_URL")); try { // follow redirects until you get it right HttpRequest httpRequest; HttpMessage httpGet; String url = baseURL + site.getString("LOGIN_URL"); while (true) { httpGet = new HttpMessage(HttpType.GET); httpGet.setUrl(url); httpGet.initializeDefaultHeaders(site); httpGet.addCookieHeaders(cookies); httpRequest = new HttpRequest(httpGet); if (httpRequest.getStatusCode() != 200) { getTempCookies(httpRequest.getHeaders()); // Find the header I want boolean found = false; for (Header h : httpRequest.getHeaders()) { if (h.getName().equals("Location")) { url = h.getValue(); found = true; } } if (!found) { throw new TrawlException("Redirect loop."); } } else { break; } } // Get headers ArrayList<Header> headers = httpRequest.getHeaders(); // Parse the cookies getTempCookies(headers); String body = httpRequest.getHtml(); Document doc = Jsoup.parse(body); Elements logins = doc.getElementsByAttributeValue("action", site.getString("LOGIN_SUBMIT")); if (logins.isEmpty()) { logins = doc.getElementsByAttributeValue("action", site.getString("BASE_URL") + site.getString("LOGIN_SUBMIT")); } if (logins.isEmpty()) { logins = doc.getElementsByAttributeValue("method", "POST"); } if (logins.isEmpty()) { throw new TrawlException("Failed to find login form!"); } if (logins.size() > 1) { logger.warn("Found multiple login forms. Picking the first one..."); } Element login = logins.get(0); // Extract the captcha image if appropriate String captchaResult = ""; if (!site.getString("CAPTCHA").equals("")) { // Download the captcha image HttpMessage getCaptcha = new HttpMessage(HttpType.GET); getCaptcha.setImage(true); if (!site.isNull("CAPTCHA_IMAGE")) { getCaptcha.setUrl(baseURL + site.getString("CAPTCHA_IMAGE")); getCaptcha.initializeDefaultImageHeaders(site); getCaptcha.addHeader(new Header("Referrer", baseURL + site.getString("LOGIN_URL"))); getCaptcha.addCookieHeaders(cookies); // Send it to deathbycaptcha SocketClient client = new SocketClient("njanetos", "2point7182"); HttpRequest image = new HttpRequest(getCaptcha); ByteArrayOutputStream os = new ByteArrayOutputStream(); ImageIO.write(image.getImage(), "png", os); Captcha result = client.decode(os.toByteArray()); captchaResult = result.toString(); } else { // Just try to get the image Elements captchas = login.getElementsByTag("img"); if (captchas.size() != 1) { throw new TrawlException( "Failed to find captcha, but the initialization file says there should be one."); } Element captchaImage = captchas.get(0); // Does it contain base64? if (captchaImage.attr("src").contains("base64")) { String src = captchaImage.attr("src").split(",")[1]; byte image[] = Base64.decodeBase64(src); ByteArrayOutputStream os = new ByteArrayOutputStream(); os.write(image); SocketClient client = new SocketClient("njanetos", "2point7182"); Captcha result = client.decode(os.toByteArray()); captchaResult = result.toString(); } else { if (captchaImage.attr("src").contains(baseURL)) { getCaptcha.setUrl(captchaImage.attr("src")); } else { getCaptcha.setUrl(baseURL + captchaImage.attr("src")); } getCaptcha.initializeDefaultImageHeaders(site); getCaptcha.addHeader(new Header("Referrer", baseURL + site.getString("LOGIN_URL"))); getCaptcha.addCookieHeaders(cookies); // Send it to deathbycaptcha SocketClient client = new SocketClient("njanetos", "2point7182"); HttpRequest image = new HttpRequest(getCaptcha); ByteArrayOutputStream os = new ByteArrayOutputStream(); ImageIO.write(image.getImage(), "png", os); Captcha result = client.decode(os.toByteArray()); captchaResult = result.toString(); } } logger.info("Decoded captcha: " + captchaResult); } // Grab any hidden fields Elements hidden = login.getElementsByAttributeValue("type", "hidden"); // Build the post response HttpMessage httpPost = new HttpMessage(HttpType.POST); httpPost.initializeDefaultHeaders(site); httpPost.addCookieHeaders(cookies); // TODO: Read this from the html! httpPost.setUrl(baseURL + site.getString("LOGIN_SUBMIT")); httpPost.appendContent(site.getString("USERNAME_FIELD"), site.getString("USERNAME")); httpPost.appendContent(site.getString("PASSWORD_FIELD"), site.getString("PASSWORD")); if (!captchaResult.equals("")) { httpPost.appendContent(site.getString("CAPTCHA_FIELD"), captchaResult); } for (int i = 0; i < hidden.size(); ++i) { httpPost.appendContent(hidden.get(i).attr("name"), hidden.get(i).attr("value")); } // Add the submit info Element submit = login.getElementsByAttributeValue("type", "submit").get(0); httpPost.appendContent(submit.attr("name"), submit.attr("value")); // Add the referrer httpPost.addHeader(new Header("Referer", baseURL + site.getString("LOGIN_URL"))); // Log in HttpRequest response = new HttpRequest(httpPost); headers = response.getHeaders(); // Add any relevant cookies getTempCookies(headers); logger.info("Successfully logged in, response code: " + response.getStatusCode()); // Were we redirected? If so, visit the redirection URL before continuing. if (response.getStatusCode() == 302) { // Send a GET request to the redirection URL before continuing. httpGet = new HttpMessage(HttpType.GET); httpGet.initializeDefaultHeaders(site); httpGet.addHeader(new Header("Referer", baseURL + site.getString("LOGIN_URL"))); String redirectionURL = getRedirectionURL(headers); httpGet.setUrl(redirectionURL); httpGet.addCookieHeaders(cookies); httpRequest = new HttpRequest(httpGet); logger.debug("Visited redirected page. Status code " + httpRequest.getStatusCode()); } } catch (ConnectionException | MalformedURLException | ProtocolException ex) { // Did not successfully log in logger.error(ex.getMessage()); return false; } catch (IOException ex) { // Did not successfully log in logger.error(ex.getMessage()); return false; } catch (Exception | InterruptedException ex) { // Did not successfully log in logger.error(ex.getMessage()); return false; } // Did we successfully log in? Then return true. return true; }
From source file:info.dolezel.fatrat.plugins.UloztoDownload.java
@Override public void processLink(String link) { //if (link.contains("/live/")) // link = link.replace("/live/", "/"); if (link.startsWith("http://uloz.to") || link.startsWith("https://uloz.to")) link = link.replace("https?://uloz.to", "https://www.uloz.to"); if (link.startsWith("http://m.uloz.to") || link.startsWith("https://m.uloz.to")) link = link.replace("https?://m.uloz.to", "https://www.uloz.to"); if (!logIn(link)) return;//from w ww .j a v a 2s .co m final String downloadLink = link; // I can't make 'link' final fetchPage(link, new PageFetchListener() { @Override public void onCompleted(ByteBuffer buf, Map<String, String> headers) { try { if (headers.containsKey("location")) { String location = headers.get("location"); if (location.contains("smazano") || location.contains("nenalezeno")) setFailed("The file has been removed"); else processLink(location); return; } CharBuffer cb = charsetUtf8.decode(buf); if (cb.toString().contains("?disclaimer=1")) { processLink(downloadLink + "?disclaimer=1"); return; } final Document doc = Jsoup.parse(cb.toString()); final Element freeForm = doc.getElementById("frm-download-freeDownloadTab-freeDownloadForm"); final Element premiumLink = doc.getElementById("#quickDownloadButton"); boolean usePremium = usePremium(downloadLink); if (cb.toString().contains("Nem dostatek kreditu")) setMessage("Credit depleted, using FREE download"); else if (usePremium && premiumLink != null) { String msg = "Using premium download"; Elements aCredits = doc.getElementsByAttributeValue("href", "/kredit"); if (!aCredits.isEmpty()) msg += " (" + aCredits.get(0).ownText() + " left)"; setMessage(msg); startDownload("http://www.uloz.to" + premiumLink.attr("href")); return; } else if (loggedIn) setMessage("Login failed, using FREE download"); Elements aNames = doc.getElementsByClass("jsShowDownload"); if (!aNames.isEmpty()) reportFileName(aNames.get(0).ownText()); final PostQuery pq = new PostQuery(); final Map<String, String> hdr = new HashMap<String, String>(); Elements eHiddens = freeForm.select("input[type=hidden]"); hdr.put("X-Requested-With", "XMLHttpRequest"); hdr.put("Referer", downloadLink); hdr.put("Accept", "application/json, text/javascript, */*; q=0.01"); for (Element e : eHiddens) pq.add(e.attr("name"), e.attr("value")); fetchPage("https://uloz.to/reloadXapca.php?rnd=" + Math.abs(new Random().nextInt()), new PageFetchListener() { @Override public void onCompleted(ByteBuffer buf, Map<String, String> headers) { CharBuffer cb = charsetUtf8.decode(buf); String captchaUrl; try { JSONObject json = new JSONObject(cb.toString()); captchaUrl = "https:" + json.getString("image"); pq.add("hash", json.getString("hash")); pq.add("timestamp", "" + json.getInt("timestamp")); pq.add("salt", "" + json.getInt("salt")); } catch (JSONException e) { setFailed("Error parsing captcha JSON"); return; } solveCaptcha(captchaUrl, new CaptchaListener() { @Override public void onFailed() { setFailed("Failed to decode the captcha code"); } @Override public void onSolved(String text) { String action = freeForm.attr("action"); pq.add("captcha_value", text); fetchPage("https://www.uloz.to" + action, new PageFetchListener() { @Override public void onCompleted(ByteBuffer buf, Map<String, String> headers) { try { CharBuffer cb = charsetUtf8.decode(buf); JSONObject obj = new JSONObject(cb.toString()); startDownload(obj.getString("url")); } catch (Exception e) { setFailed("" + e); } } @Override public void onFailed(String error) { setFailed(error); } }, pq.toString(), hdr); } }); } @Override public void onFailed(String error) { setFailed("Failed to load captcha AJAX page"); } }); } catch (Exception e) { e.printStackTrace(); setFailed(e.toString()); } } @Override public void onFailed(String error) { setFailed("Failed to load the initial page"); } }, null); }