Java tutorial
/* * Copyright 2012-2014 Battams, Derek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.schedulesdirect.api; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; import java.nio.file.FileSystem; import java.nio.file.FileSystemAlreadyExistsException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.schedulesdirect.api.exception.InvalidJsonObjectException; import org.schedulesdirect.api.exception.JsonEncodingException; import org.schedulesdirect.api.utils.UriUtils; import com.fasterxml.jackson.core.JsonParseException; /** * <p>An implementation of EpgClient that uses a local zip file as its data source</p> * * <p>The zip file to be used must follow a specific format and structure. Such a zip file * can be generated by running the EPG Grabber application.</p> * * <p>This implementation has two common uses:</p> * <ol> * <li>For development and testing of this API, you can download a zip file of listings data and reuse it instead of constantly hitting the Schedules Direct servers.</li> * <li>For production, applications <b>should always</b> simply download a zip file once a day and reuse that zip file via this client implentation, which provides a simple form a caching.</li> * </ol> * * <p> * Most every real world application should only be accessing EPG data via instances of this class. Apps should * download their EPG data once a day using the sdjson grabber application then feeding that generated zip file * into instances of this class to access the EPG data in their apps. * </p> * @author Derek Battams <derek@battams.ca> * */ public class ZipEpgClient extends EpgClient { static private final Log LOG = LogFactory.getLog(ZipEpgClient.class); /** * Name of the file holding the zip file version number */ static public final String ZIP_VER_FILE = "version.txt"; /** * The zip file version this grabber generates */ static public final int ZIP_VER = 10; /** * The default charset encoding used for all data in the generated zip file */ static public final Charset ZIP_CHARSET = Charset.forName("UTF-8"); /** * The file containing all the lineups stored in this zip cache */ static public final String LINEUPS_LIST = "lineups.txt"; /** * The file containing the user data for this zip cache (i.e. the user who generated the cache) */ static public final String USER_DATA = "user.txt"; static private final Map<String, AtomicInteger> CLNT_COUNT = Collections .synchronizedMap(new HashMap<String, AtomicInteger>()); static private String getSrcZipKey(File src) { return src.getAbsolutePath(); } /** * Regex of invalid chars for file names in the zip */ static public final String INVALID_FILE_CHARS = "[\\s\\\\\\/:\\*\\?\"<>\\|]"; /** * Scrub a file name, replacing invalid chars * @param input The file name to scrub * @return The scrubbed file name; suitable for use in the generated zip file */ static public String scrubFileName(String input) { return input.replaceAll(INVALID_FILE_CHARS, "_"); } /** * Returns the first 10 characters of a programs ID * @param input The program ID * @return The first 10 characters */ static public String artworkId(String input) { String ret = input; if (ret.length() > 10) { ret = ret.substring(0, 10); } return ret; } private File src; private FileSystem vfs; private Map<String, Lineup> lineups; private Map<String, Program> progCache; private Map<String, List<Artwork>> artCache; private boolean closed; private boolean detailsFetched; /** * Constructor * @param zip The zip path to be used as the data source for this client implementation * @param baseUrl The base URL used to construct absolute URLs from relative URL data in the raw JSON * @throws IOException Thrown on any IO error reading the zip file */ public ZipEpgClient(final Path zip, final String baseUrl) throws IOException { this(zip.toFile(), baseUrl); } /** * Constructor * @param zip The zip file to be used as the data source for this client implementation * @param baseUrl The base URL used to construct absolute URLs from relative URL data in the raw JSON * @throws IOException Thrown on any IO error reading the zip file */ public ZipEpgClient(final File zip, final String baseUrl) throws IOException { super(null, baseUrl); src = zip; progCache = new HashMap<String, Program>(); artCache = new HashMap<>(); URI fsUri; try { fsUri = new URI(String.format("jar:%s", zip.toURI())); } catch (URISyntaxException e1) { throw new RuntimeException(e1); } try { try { this.vfs = FileSystems.newFileSystem(fsUri, Collections.<String, Object>emptyMap()); } catch (FileSystemAlreadyExistsException e) { this.vfs = FileSystems.getFileSystem(fsUri); } Path verFile = vfs.getPath(ZIP_VER_FILE); if (Files.exists(verFile)) { try (InputStream ins = Files.newInputStream(verFile)) { int ver = Integer.parseInt(IOUtils.toString(ins, ZIP_CHARSET.toString())); if (ver != ZIP_VER) throw new IOException( String.format("Zip file is not expected version! [v=%d; e=%d]", ver, ZIP_VER)); } } else throw new IOException(String.format("Zip file of version %d required!", ZIP_VER)); LOG.debug(String.format("Zip file format validated! [version=%d]", ZIP_VER)); lineups = new HashMap<String, Lineup>(); try (InputStream ins = Files.newInputStream(vfs.getPath(LINEUPS_LIST))) { String input = IOUtils.toString(ins, ZIP_CHARSET.toString()); JSONObject o; try { o = Config.get().getObjectMapper().readValue(input, JSONObject.class); } catch (JsonParseException e) { throw new JsonEncodingException(String.format("ZipLineups: %s", e.getMessage()), e, input); } try { JSONArray lineups = o.getJSONArray("lineups"); for (int i = 0; i < lineups.length(); ++i) { JSONObject l = lineups.getJSONObject(i); this.lineups.put(l.getString("uri"), new Lineup(l.getString("name"), l.getString("location"), l.getString("uri"), l.getString("transport"), this)); } } catch (JSONException e) { throw new InvalidJsonObjectException(String.format("ZipLineups: %s", e.getMessage()), e, o.toString(3)); } } String vfsKey = getSrcZipKey(zip); AtomicInteger i = CLNT_COUNT.get(vfsKey); if (i == null) { i = new AtomicInteger(0); CLNT_COUNT.put(vfsKey, i); } i.incrementAndGet(); closed = false; detailsFetched = false; } catch (Throwable t) { if (vfs != null) try { close(); } catch (IOException e) { LOG.error("IOError closing VFS!", e); } throw t; } } /** * Constructor * @param zip The zip path to be used as the data source for this client implementation * @throws IOException Thrown on any IO error reading the zip file */ public ZipEpgClient(final Path zip) throws IOException { this(zip.toFile()); } /** * Constructor * @param zip The zip file to be used as the data source for this client implementation * @throws IOException Thrown on any IO error reading the zip file */ public ZipEpgClient(final File zip) throws IOException { this(zip, null); } @Override public UserStatus getUserStatus() throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); String input = null; try (InputStream ins = Files.newInputStream(vfs.getPath(USER_DATA))) { input = IOUtils.toString(ins, ZIP_CHARSET.toString()); return new UserStatus(Config.get().getObjectMapper().readValue(input, JSONObject.class), null, this); } catch (JsonParseException e) { throw new JsonEncodingException(String.format("ZipUser: %s", e.getMessage()), e, input); } } @Override public void close() throws IOException { if (!closed) { purgeCache(); String vfsKey = getSrcZipKey(src); AtomicInteger i = CLNT_COUNT.get(vfsKey); int v = i != null ? i.decrementAndGet() : 0; if (v <= 0) { LOG.debug("Calling close() for " + vfsKey); vfs.close(); } else if (LOG.isDebugEnabled()) LOG.debug(String.format("Skipped close() for %s; c=%d", vfsKey, i != null ? i.get() : Integer.MIN_VALUE)); closed = true; } } @Override protected void finalize() throws Throwable { super.finalize(); CLNT_COUNT.get(getSrcZipKey(src)).set(0); close(); } @Override protected Airing[] fetchSchedule(final Station station) throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); List<Airing> airs = new ArrayList<>(); Path path = vfs.getPath(String.format("schedules/%s.txt", scrubFileName(station.getId()))); if (Files.exists(path)) { String input = null; JSONObject o = null; try (InputStream ins = Files.newInputStream(path)) { input = IOUtils.toString(ins, ZIP_CHARSET.toString()); try { o = Config.get().getObjectMapper().readValue(input, JSONObject.class); } catch (JsonParseException e) { throw new JsonEncodingException( String.format("Schedule[%s]: %s", station.getId(), e.getMessage()), e, input); } JSONArray jarr = o.getJSONArray("programs"); for (int i = 0; i < jarr.length(); ++i) { JSONObject src = jarr.getJSONObject(i); Program p = fetchProgram(src.getString("programID")); if (p != null) airs.add(new Airing(src, p, station)); } } catch (JSONException e) { throw new InvalidJsonObjectException( String.format("Schedule[%s]: %s", station.getId(), e.getMessage()), e, o.toString(3)); } } else if (LOG.isDebugEnabled()) LOG.debug("Requested schedule not available in cache: " + station.getId()); return airs.toArray(new Airing[0]); } @Override protected Program fetchProgram(final String progId) throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); Program p = progCache.get(progId); if (p == null) { Path path = vfs.getPath(String.format("programs/%s.txt", scrubFileName(progId))); if (!Files.exists(path) && progId.startsWith("SH")) { path = vfs.getPath(String.format("seriesInfo/%s.txt", scrubFileName(progId))); } if (Files.exists(path)) { try (InputStream ins = Files.newInputStream(path)) { String data = IOUtils.toString(ins, ZIP_CHARSET.toString()); if (data != null) { JSONObject obj; try { obj = Config.get().getObjectMapper().readValue(data, JSONObject.class); } catch (JsonParseException e) { throw new JsonEncodingException( String.format("ZipProgram[%s]: %s", progId, e.getMessage()), e, data); } String cachedMd5 = obj.optString("md5", ""); if (cachedMd5 != null && !"".equals(cachedMd5)) { p = new Program(obj, this); progCache.put(progId, p); } } } catch (JSONException e) { throw new IOException("JSON error!", e); } } } return p; } @Override protected Artwork[] fetchArtwork(String progId) throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); String aId = artworkId(progId); List<Artwork> artworks = artCache.get(aId); if (artworks == null) { artworks = new ArrayList<>(); artCache.put(aId, artworks); Path path = vfs.getPath(String.format("artwork/%s.txt", aId)); if (Files.exists(path)) { try (InputStream ins = Files.newInputStream(path)) { String data = IOUtils.toString(ins, ZIP_CHARSET.toString()); if (data != null) { JSONObject artworkInfo; try { artworkInfo = Config.get().getObjectMapper().readValue(data, JSONObject.class); Object temp = artworkInfo.get("data"); if (temp instanceof JSONArray) { JSONArray artworkArr = artworkInfo.getJSONArray("data"); for (int i = 0; i < artworkArr.length(); ++i) { JSONObject awObj = artworkArr.getJSONObject(i); artworks.add(new Artwork(awObj, this)); } } } catch (JsonParseException e) { throw new JsonEncodingException( String.format("ZipProgram[%s]: %s", progId, e.getMessage()), e, data); } } } catch (JSONException e) { throw new IOException("JSON error!", e); } } } return artworks.toArray(new Artwork[0]); } @Override protected Map<Station, Airing[]> fetchSchedules(final Lineup lineup) throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); Map<Station, Airing[]> scheds = new HashMap<Station, Airing[]>(); for (Station s : lineup.getStations()) scheds.put(s, fetchSchedule(s)); return scheds; } @Override protected Map<String, Program> fetchPrograms(final String[] progIds) throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); Map<String, Program> progs = new HashMap<String, Program>(); for (String id : progIds) progs.put(id, fetchProgram(id)); return progs; } /** * Find the metadata object for the given device name * @param metas The array of metadata objects * @param dev The device name being sought after * @return The metadata object for the device name or null if it could not be found * @throws JSONException On any JSON error encountered */ protected JSONObject findMetadataForDevice(final JSONArray metas, final String dev) throws JSONException { if (closed) throw new IllegalStateException("Instance has already been closed!"); JSONObject val = null; for (int i = 0; i < metas.length(); ++i) { JSONObject o = metas.getJSONObject(i); if (dev.equals(o.optString("device"))) { val = o; break; } } return val; } @Override public Lineup[] getLineups() throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); if (!detailsFetched) { for (Lineup l : lineups.values()) l.fetchDetails(true); detailsFetched = true; } return lineups.values().toArray(new Lineup[0]); } @Override public void purgeCache() { if (closed) throw new IllegalStateException("Instance has already been closed!"); progCache.clear(); artCache.clear(); } @Override public void purgeCache(final Object obj) { if (closed) throw new IllegalStateException("Instance has already been closed!"); if (obj instanceof Program) progCache.remove(((Program) obj).getId()); } @Override public void deleteMessage(final Message msg) throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); throw new UnsupportedOperationException("Messages can only be deleted via the NetworkEpgClient!"); } /** * Check all downloaded schedules for gaps in the airing schedules * <p> * The data received from upstream should never have gaps in the schedules. If so, that * is an error that invalidates the data in the local cache and the data should no longer * be trusted. * </p> * @return Null if no gaps are found in any schedules in the local cache or a two element array of Airing objects representing the first gap found * @throws IOException in case of any IO error during the operation */ public Airing[] findScheduleGap() throws IOException { for (Lineup l : getLineups()) for (Station s : l.getStations()) { Airing prev = null; for (Airing a : s.getAirings()) if (prev == null || new Date(prev.getGmtStart().getTime() + 1000L * prev.getDuration()) .equals(a.getGmtStart())) prev = a; else return new Airing[] { prev, a }; } return null; } @Override public SystemStatus getSystemStatus() throws IOException { if (closed) throw new IllegalStateException("Instance has already been closed!"); try (InputStream ins = Files.newInputStream(vfs.getPath(USER_DATA))) { String input = IOUtils.toString(ins, ZIP_CHARSET.toString()); JSONObject user; try { user = Config.get().getObjectMapper().readValue(input, JSONObject.class); } catch (JsonParseException e) { throw new JsonEncodingException(String.format("ZipSysStatus: %s", e.getMessage()), e, input); } return new SystemStatus(user.getJSONArray("systemStatus")); } } @Override protected InputStream fetchLogoStream(final Station station) throws IOException { String url = station.getLogo().getUrl().toString(); String ext = url.substring(url.lastIndexOf('.') + 1); Path p = vfs.getPath(String.format("logos/%s.%s", station.getCallsign(), ext)); if (Files.exists(p)) { ByteArrayOutputStream os = new ByteArrayOutputStream(); try (InputStream ins = Files.newInputStream(p)) { IOUtils.copy(ins, os); return new ByteArrayInputStream(os.toByteArray()); } } else return null; } @Override public int registerLineup(final String path) throws IOException { throw new UnsupportedOperationException("Unsupported operation"); } @Override public int unregisterLineup(final Lineup l) throws IOException { throw new UnsupportedOperationException("Unsupported operation"); } // Ignore the search parameters and just return whatever's in the cache file @Override protected Lineup[] searchForLineups(String location, String zip) throws IOException { return getLineups(); } @Override public Lineup getLineupByUriPath(String path) throws IOException { for (Lineup l : getLineups()) { if (l.getUri().equals(UriUtils.stripApiVersion(path))) return l; } return null; } @Override protected String fetchChannelMapping(Lineup lineup) throws IOException { try (InputStream ins = Files.newInputStream( vfs.getPath("maps", ZipEpgClient.scrubFileName(String.format("%s.txt", lineup.getId()))))) { String input = IOUtils.toString(ins, ZipEpgClient.ZIP_CHARSET.toString()); try { return Config.get().getObjectMapper().readValue(input, JSONObject.class).toString(); } catch (JsonParseException e) { throw new JsonEncodingException(String.format("ZipLineupMap: %s", e.getMessage()), e, input); } } } }