public class DatabaseHelper extends OrmLiteSqliteOpenHelper {
	public static final String LOG_TAG = "DatabaseHelper";
	
	public static final String DATABASE_NAME = "ka.sqlite3";
	public static final int DATABASE_RESOURCE_ID = R.raw.db;
	public static final int DATABASE_VERSION = 120; // analagous to 1.2.0 release
	
	private Context context;
	private Dao<Video, String> videoDao;
	private Dao<Topic, String> topicDao;
	private Dao<User, String> userDao;
	private Dao<UserVideo, Integer> userVideoDao;
	private StringBuilder stringBuilder = new StringBuilder();
	private Formatter stringFormatter = new Formatter(stringBuilder);
	
	public static enum Type {
		USER(User.class),
		USERVIDEO(UserVideo.class),
		VIDEO(Video.class),
		TOPIC(Topic.class);
		
		public final Class<?> cls;
		
		Type(Class<?> cls) {
			this.cls = cls;
		}
	}
	
	public interface Callbacks {
		public abstract void onChange();
	}
	
	private ArrayList<Callbacks> callbacks = new ArrayList<Callbacks>();
	
	public void registerCallbacks(Callbacks callbacks) {
		this.callbacks.add(callbacks);
	}
	
	public boolean unregisterCallbacks(Callbacks callbacks) {
		return this.callbacks.remove(callbacks);
	}

	public DatabaseHelper(Context context) {
	    super(context, DATABASE_NAME, null, DATABASE_VERSION,
	        R.raw.ormlite_config);
		this.context = context;
	}
	
	private boolean databaseExists() {
		for (String name : context.databaseList()) {
			if (name.equals(getDatabaseName())) {
				return true;
			}
		}
		return false;
	}
	
	private void copyRawDatabase() {
		new DatabaseImporter(context).import_(DATABASE_RESOURCE_ID, getDatabaseName());
	}
	
	@Override
	public SQLiteDatabase getWritableDatabase() {
		if (!databaseExists()) {
			copyRawDatabase();
		}
		return super.getWritableDatabase();
	}

	@Override
	public SQLiteDatabase getReadableDatabase() {
		if (!databaseExists()) {
			copyRawDatabase();
		}
		return super.getReadableDatabase();
	}

	@Override
	public void onCreate(SQLiteDatabase database, ConnectionSource connectionSource) {
		
	}
	

	@Override
	public void onUpgrade(final SQLiteDatabase database,
			final ConnectionSource connectionSource, final int oldVersion, final int newVersion) {
		
		
		if (oldVersion < 111) {
			// Create new Caption table.
			try {
				TableUtils.createTableIfNotExists(connectionSource, Caption.class);
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
		
		if (oldVersion < 114) {
			// Could instead create an ORMLite pojo for topicvideo and do this with TableUtils.
			database.execSQL("CREATE TABLE IF NOT EXISTS `topicvideo` (`_id` INTEGER PRIMARY KEY AUTOINCREMENT, `topic_id` VARCHAR, `video_id` VARCHAR )");
		}
		
		if (newVersion >= 111) {
			
			// Played with the idea of putting this all into an AsyncTask, but
			//  1) Can crash the app. For example, in 114 upgrade we add topicvideo table; if it doesn't exist and user reaches a video list, we 
			//     crash. Worse, if we do crash here the app is broken. Db version has been bumped, so this code doesn't get another chance to run.
			//  2) To prevent crash, tried showing AlertDialog. The context we get here is "not an application context", so trying to show a window
			//     (or sometimes a toast) crashes us with a null window token error. I dug deep to try to pass in a useful context with no luck.
			//     See branch temp/pass_context_to_helper.
			//  3) It doesn't take very long (truncate / reload full video and topic tables is just a few seconds, not long enough for ANR on Fire HD)
			
			new DatabaseImporter(context).import_(DATABASE_RESOURCE_ID, "temp");
			SQLiteDatabase tempDb = new TempHelper(context).getReadableDatabase();
			
			if (oldVersion < 111) {
				do111Upgrade(database, tempDb);
			}
			
			if (oldVersion < 114 && newVersion >= 114) {
				// add topicvideo table
				do114Upgrade(database, tempDb);
			}
			
			if (oldVersion < 120 && newVersion >= 120) {
				// add video.dlm_id column
				do120Upgrade(database, tempDb);
			}
			
			tempDb.close();
			context.deleteDatabase(TempHelper.DB_NAME);
			
			// Update download status from storage. This recovers any lost during library updates in 1.1.3.
			database.beginTransaction();
			try {
				for (String yid : getAllDownloadedYoutubeIds()) {
					database.execSQL("update video set download_status=? where youtube_id=?", new Object[] {Video.DL_STATUS_COMPLETE, yid});
				}
				database.setTransactionSuccessful();
			} finally {
				database.endTransaction();
			}
					
		}
			
	}

	private void do111Upgrade(SQLiteDatabase database, SQLiteDatabase tempDb) {
		Log.d(LOG_TAG, "do111Upgrade");
		
		Cursor c;
			
		// Maintain download status.
		tempDb.beginTransaction();
		try {
			String sql = "select max(download_status), youtube_id from video where download_status != ? group by youtube_id";
			c = database.rawQuery(sql, new String[] {"" + Video.DL_STATUS_NOT_STARTED});
			while (c.moveToNext()) {
				String youtube_id = c.getString(c.getColumnIndex("youtube_id"));
				String download_status = c.getString(c.getColumnIndex("max(download_status)"));
				ContentValues v = new ContentValues();
				v.put("download_status", download_status);
				String[] idArg = new String[] {youtube_id};
				tempDb.update("video", v, "youtube_id = ?", idArg);
				
				// cursor over parent topics of this video
				sql = "select topic._id from topic, topicvideo, video where video.youtube_id=? and topicvideo.video_id=video.readable_id and topicvideo.topic_id=topic._id";
				Cursor c1 = tempDb.rawQuery(sql, idArg);
				Log.d(LOG_TAG, String.format("updating counts for %d topics", c1.getCount()));
				while (c1.moveToNext()) {
					String topicId = c1.getString(c1.getColumnIndex("_id"));
					incrementDownloadedVideoCounts(tempDb, topicId, "topic");
				}
				c1.close();
			}
			c.close();
			tempDb.setTransactionSuccessful();
		} finally {
			tempDb.endTransaction();
		}
			
		// Migrate topics.
		database.beginTransaction();
		try {
			List<String> schema = new ArrayList<String>();
			c = tempDb.rawQuery("select sql from sqlite_master where tbl_name=?", new String[] {"topic"});
			while (c.moveToNext()) {
				String s = c.getString(0);
				if (s != null) {
					schema.add(s);
				}
			}
			c.close();
			database.execSQL("drop table topic");
			for (String s : schema) {
				Log.d(LOG_TAG, s);
				database.execSQL(s);
			}
			
			c = tempDb.rawQuery("select * from topic", null);
			while (c.moveToNext()) {
				ContentValues v = new ContentValues();
				DatabaseUtils.cursorRowToContentValues(c, v);
				database.insert("topic", null, v);
			}
			c.close();
			database.setTransactionSuccessful();
		} finally {
			database.endTransaction();
		}
		// Migrate videos.
		database.beginTransaction();
		try {
			List<String> schema = new ArrayList<String>();
			c = tempDb.rawQuery("select sql from sqlite_master where tbl_name=?", new String[] {"video"});
			while (c.moveToNext()) {
				String s = c.getString(0);
				if (s != null) {
					schema.add(s);
				}
			}
			c.close();
			database.execSQL("drop table video");
			for (String s : schema) {
				Log.d(LOG_TAG, s);
				database.execSQL(s);
			}
			
			c = tempDb.rawQuery("select * from video", null);
			while (c.moveToNext()) {
				ContentValues v = new ContentValues();
				DatabaseUtils.cursorRowToContentValues(c, v);
				database.insert("video", null, v);
			}
			c.close();
			database.setTransactionSuccessful();
		} finally {
			database.endTransaction();
		}
		
		Log.d(LOG_TAG, "do111Upgrade returning");
	}
	
	private void do114Upgrade(SQLiteDatabase database, SQLiteDatabase tempDb) {
		Log.d(LOG_TAG, "do114Upgrade");
		
		database.beginTransaction();
		Cursor c = null;
		try {
			List<String> schema = new ArrayList<String>();
			c = tempDb.rawQuery("select sql from sqlite_master where tbl_name=?", new String[] {"topicvideo"});
			while (c.moveToNext()) {
				String s = c.getString(0);
				if (s != null) {
					schema.add(s);
				}
			}
			c.close();
			database.execSQL("drop table topicvideo");
			for (String s : schema) {
				Log.d(LOG_TAG, s);
				database.execSQL(s);
			}
			
			c = tempDb.rawQuery("select * from topicvideo", null);
			while (c.moveToNext()) {
				ContentValues v = new ContentValues();
				DatabaseUtils.cursorRowToContentValues(c, v);
				database.insert("topicvideo", null, v);
			}
			c.close();
			database.setTransactionSuccessful();
		} finally {
			if (c != null && !c.isClosed()) {
				c.close();
			}
			database.endTransaction();
		}
		
		Log.d(LOG_TAG, "do114Upgrade returning");
	}
	
	private void do120Upgrade(SQLiteDatabase database, SQLiteDatabase tempDb) {
		// adds dlm_id column
		
		
		// Move existing video data into temp table (dlm_id will be left at default of 0).
		tempDb.beginTransaction();
		Cursor c = null;
		try {
			tempDb.execSQL("delete from video");
			c = database.rawQuery("select * from video", null);
			while (c.moveToNext()) {
				ContentValues v = new ContentValues();
				DatabaseUtils.cursorRowToContentValues(c, v);
				tempDb.insert("video", null, v);
			}
			tempDb.setTransactionSuccessful();
		} finally {
			if (c != null && !c.isClosed()) {
				c.close();
			}
			tempDb.endTransaction();
		}
			
		// Recreate main video table with the new column.
		database.execSQL("drop table video");
		c = tempDb.rawQuery("select sql from sqlite_master where tbl_name=?", new String[] {"video"});
		while (c.moveToNext()) {
			String s = c.getString(0);
			if (s != null) {
				database.execSQL(s);
			}
		}
		c.close();
		
		// Move data from temp back into main db.
		database.beginTransaction();
		try {
			c = tempDb.rawQuery("select * from video", null);
			while (c.moveToNext()) {
				ContentValues v = new ContentValues();
				DatabaseUtils.cursorRowToContentValues(c, v);
				database.insert("video", null, v);
			}
			database.setTransactionSuccessful();
		} finally {
			if (c != null && !c.isClosed()) {
				c.close();
			}
			database.endTransaction();
		}
		
		syncWithDownloadManager();
	}
	
	class TempHelper extends OrmLiteSqliteOpenHelper {
		
		static final String DB_NAME = "temp";

		public TempHelper(Context context) {
		    super(context, DB_NAME, null, 1);
		}

		@Override
		public void onCreate(SQLiteDatabase database, ConnectionSource connectionSource) {
		}

		@Override
		public void onUpgrade(SQLiteDatabase database,
				ConnectionSource connectionSource, int oldVersion, int newVersion) {
		}
	}
	
	
	///////////////////////////////  Download-related  ////////////////////////////////
	
	public void syncWithDownloadManager() {
		Log.w(LOG_TAG, "syncWithDownloadManager");
		DownloadManager mgr = (DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE);
		DownloadManager.Query q = new DownloadManager.Query();
		
		class DownloadInfo {
			DownloadInfo(long id, int status) {
				this.id = id;
				this.status = status;
			}
			final long id;
			final int status;
		}
		
		// Query for all.
		// In case the same video has been downloaded more than once, we want to use the most
		// optimistic one. So on the first pass, set dlm_id and hash youtubeIds against their status,
		// overwriting failed with in progress, and in progress with complete.
		Cursor c = mgr.query(q);
		final Map<String, DownloadInfo> infos = new HashMap<String, DownloadInfo>();
		while (c.moveToNext()) {
			String path = c.getString(c.getColumnIndex(DownloadManager.COLUMN_LOCAL_FILENAME));
			if (path == null) {
				//weird.
				continue;
			}
			String youtubeId = OfflineVideoManager.youtubeIdFromFilename(path);
			long dlm_id = c.getLong(c.getColumnIndex(DownloadManager.COLUMN_ID));
			int status = c.getInt(c.getColumnIndex(DownloadManager.COLUMN_STATUS));
			boolean exists = new File(path).exists();
			boolean shouldExist = 0 != (status & (STATUS_PAUSED | STATUS_RUNNING | STATUS_SUCCESSFUL));
			if (!exists && shouldExist) {
				status = STATUS_FAILED;
			}
				
				
			DownloadInfo info = new DownloadInfo(dlm_id, status);
			DownloadInfo existing = infos.get(youtubeId);
			if (existing == null
					|| existing.status == DownloadManager.STATUS_FAILED
					|| existing.status != DownloadManager.STATUS_SUCCESSFUL && info.status == DownloadManager.STATUS_SUCCESSFUL) {
				infos.put(youtubeId, info);
			}
		}
		c.close();
		
		// Now get all the matching videos.
		QueryBuilder<Video, String> qb = videoDao.queryBuilder();
		List<Video> tempVideos = null;
		try {
			qb.where().in("youtube_id", infos.keySet().toArray(new Object[0]));
			tempVideos = qb.query();
		} catch (SQLException e) {
			e.printStackTrace();
		}
		
		// On the second pass, do the actual updates based on the best completion status we found.
		if (tempVideos != null) {
			final List<Video> videos = tempVideos;
			
			try {
				videoDao.callBatchTasks(new Callable<Integer>() {
					@Override public Integer call() {
						int result = 0;
						
						for (Video video : videos) {
							result++;
							DownloadInfo info = infos.get(video.getYoutube_id());
							video.setDlm_id(info.id);
							
							int status = info.status;
							byte newStatus = -1;
							switch (status) {
							case DownloadManager.STATUS_FAILED:
								removeDownloadFromDownloadManager(video);
								continue;
							case DownloadManager.STATUS_PENDING:
							case DownloadManager.STATUS_PAUSED:
							case DownloadManager.STATUS_RUNNING:
								newStatus = Video.DL_STATUS_IN_PROGRESS;
								break;
							case DownloadManager.STATUS_SUCCESSFUL:
								newStatus = Video.DL_STATUS_COMPLETE;
								break;
							}
							video.setDownload_status(newStatus);
							try {
								videoDao.update(video);
							} catch (SQLException e) {
								e.printStackTrace();
							}
						}
						
						return result;
					}
				});
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
		Log.w(LOG_TAG, "  <--   syncWithDownloadManager");
	}
	
	public Video getVideoForFilename(String path) {
		if (path == null) return null;
		
		String youtubeId = OfflineVideoManager.youtubeIdFromFilename(path);
		QueryBuilder<Video, String> q = videoDao.queryBuilder();
		try {
			q.where().eq("youtube_id", youtubeId);
			return q.queryForFirst();
		} catch (SQLException e) {
			e.printStackTrace();
			return null;
		}
	}

	public long removeDownloadFromDownloadManager(Video video) {
		if (video == null) return 0;
		
		long toRemove = video.getDlm_id();
		if (toRemove > 0) {
			((DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE)).remove(toRemove);
			resetDownloadStatusAndDlmId(toRemove);
		}
		return toRemove;
	}
	
	private void resetDownloadStatusAndDlmId(long dlm_id) {
		try {
			UpdateBuilder<Video, String> q = getVideoDao().updateBuilder();
			q.where().eq("dlm_id", dlm_id);
			q.updateColumnValue("dlm_id", 0);
			q.updateColumnValue("download_status", Video.DL_STATUS_NOT_STARTED);
			q.update();
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}
	
	////////////////////////////////////////////////////////////////////////////////////

	
	/**
	 * Increments downloaded_video_count field on the topic with the given id and all its ancestors.
	 * 
	 * TODO : review the need for this. I think it's moot, as we now calculate download counts for topics on demand.
	 * 
	 * @param db The db in which to do the incrementing.
	 * @param topicId The id of the topic to update.
	 * @param topicTableName The name of the topic table in the given db.
	 */
	public static void incrementDownloadedVideoCounts(SQLiteDatabase db, String topicId, String topicTableName) {
		Log.d(LOG_TAG, "incrementDownloadedVideoCounts: " + topicId);
		String sql = String.format("update %s set downloaded_video_count = downloaded_video_count + 1 where _id=?", topicTableName);
		String[] idArg = new String[] {topicId};
		db.execSQL(sql, idArg);
		
		sql = String.format("select parentTopic_id, downloaded_video_count from %s where _id=?", topicTableName);
		Cursor c = db.rawQuery(sql, idArg);
		c.moveToFirst();
		String parentId = c.getString(c.getColumnIndex("parentTopic_id"));
		String dlc = c.getString(c.getColumnIndex("downloaded_video_count"));
		Log.d(LOG_TAG, "  --> " + dlc);
		c.close();
		if (parentId != null) {
			incrementDownloadedVideoCounts(db, parentId, topicTableName);
		}
	}
	
	
	public Dao<UserVideo, Integer> getUserVideoDao() throws SQLException {
		if (userVideoDao == null) {
			userVideoDao = getDao(UserVideo.class);
		}
		return userVideoDao;
	}
	
	public Dao<User, String> getUserDao() throws SQLException {
		if (userDao == null) {
			userDao = getDao(User.class);
		}
		return userDao;
	}
	
	public Dao<Video, String> getVideoDao() throws SQLException {
		if (videoDao == null) {
			videoDao = getDao(Video.class);
		}
		return videoDao;
	}

	public Dao<Topic, String> getTopicDao() throws SQLException {
		if (topicDao == null) {
			topicDao = getDao(Topic.class);
		}
		return topicDao;
	}
	
	/**
	 * Build a raw SQL string finding the descendant videos of the given topic at exactly the given depth.
	 * 
	 * This returns FROM and WHERE clauses. The SELECT and other parts of the query are the responsibility
	 * of the caller. In particular, it is possible to add AND/OR clauses after the WHERE.
	 * 
	 * @param depth The tree depth at which to search for videos (0 for immediate children).
	 * @return A string with appropriate FROM and WHERE clauses.
	 */
	public String buildVideosForTopicSubQuery(int depth) {
		String from = buildTopicVideoFromClause(depth);
		String where = buildTopicVideoWhereClause(depth);
		stringFormatter.format("%s %s", from, where);
		
		String result = stringBuilder.toString();
		stringBuilder.setLength(0);
		Log.v(LOG_TAG, "returning query string: " + result);
		return result;
		
	}
	
	private String buildTopicVideoFromClause(int depth) {
		if (depth == 0) {
			stringFormatter.format("from %s", TABLE_VIDEO);
		} else {
			stringFormatter.format(", %s as p%%d", TABLE_TOPIC);
			String extra = stringBuilder.toString();
			stringBuilder.setLength(0);
			
			stringFormatter.format("from %s, %s as p0", TABLE_VIDEO, TABLE_TOPIC);
			// for each depth N, add a ", topic as pN"
			for (int i=0; i<depth-1; ++i) {
				stringFormatter.format(extra, i+1);
			}
		}
		
		String result = stringBuilder.toString();
		stringBuilder.setLength(0);
		return result;
	}
	
	private String buildTopicVideoWhereClause(int depth) {
		if (depth == 0) {
			stringFormatter.format("where %s = ?", COL_FK_TOPIC);
		} else {
			stringFormatter.format(" p%%d.%s = p%%d.%s and", COL_FK_TOPIC, COL_TOPIC_ID);
			String extraWhere = stringBuilder.toString();
			stringBuilder.setLength(0);
			
			stringFormatter.format("where %s.%s = p0.%s and", TABLE_VIDEO, COL_FK_TOPIC, COL_TOPIC_ID);
		    // for each depth N, add a "pN.parentTopic_id = pN+1._id and"
			for (int i=0; i<depth-1; ++i) {
				stringFormatter.format(extraWhere, i, i+1);
			}
			stringFormatter.format(" p%d.%s = ?", depth - 1, COL_FK_TOPIC);
		}
		
		String result = stringBuilder.toString();
		stringBuilder.setLength(0);
		return result;
	}
	
	public boolean deleteDatabase(Context context) {
		return context.deleteDatabase(DATABASE_NAME);
	}
	
	public List<Video> getAllDownloadedVideos() {
		Dao<Video, String> videoDao;
		List<Video> result;
		try {
			videoDao = getVideoDao();
			QueryBuilder<Video, String> qb = videoDao.queryBuilder();
			qb.where().ne(COL_DL_STATUS, Video.DL_STATUS_NOT_STARTED);
			result = videoDao.query(qb.prepare());
		} catch (SQLException e) {
			e.printStackTrace();
			result = new ArrayList<Video>();
		}
		return result;
	}
	
	public List<UserVideo> getUserVideos(User user, List<Video> videos) {
		if (user == null) {
			return null;
		}
		List<UserVideo> result = null;
		List<String> ids = new ArrayList<String>(videos.size());
		for (Video v : videos) {
			ids.add(v.getReadable_id());
		}
		try {
			userVideoDao = getUserVideoDao();
			QueryBuilder<UserVideo, Integer> q = userVideoDao.queryBuilder();
			q.where().in("video_id", ids);
			q.where().eq("user_id", user.getNickname());
			result = userVideoDao.query(q.prepare());
		} catch (SQLException e) {
			e.printStackTrace();
			result = new ArrayList<UserVideo>();
		}
		return result;
	}
	
	/**
	 * Update a video's download_status in the database.
	 * 
	 * Proxy to videoDao.update, but will not modify any fields but download_status.
	 * Also updates the downloaded_video_count of all Topics in the video's parent 
	 * hierarchy.
	 * 
	 * @param video The video to update.
	 */
	public void updateDownloadStatus(Video video, int was, int is) {
		Log.d(LOG_TAG, "updateDownloadStatus");
		try {
			Dao<Video, String> videoDao = getVideoDao();
			UpdateBuilder<Video, String> u = videoDao.updateBuilder();
			u.updateColumnValue("download_status", is);
			u.where().eq("youtube_id", video.getYoutube_id());
			videoDao.update(u.prepare());
			
			// DEBUG
//			QueryBuilder<Video, String> q = videoDao.queryBuilder();
//			q.where().eq("readable_id", video.getReadable_id());
//			Video v = videoDao.queryForFirst(q.prepare());
//			Log.d(LOG_TAG, String.format("updated dl status of %s from %d to %d", video.getTitle(), was, v.getDownload_status()));
			
//			Dao<Topic, String> topicDao = getTopicDao();
//			UpdateBuilder<Topic, String> ub = topicDao.updateBuilder();
//			ub.updateColumnExpression("downloaded_video_count", "downloaded_video_count + 1");
//			Topic p = video.getParentTopic();
//			while (p != null) {
//				topicDao.refresh(p);
//				Log.d(LOG_TAG, String.format("updating parent topic %s. Value was %d...", p.getTitle(), p.getDownloaded_video_count()));
//				ub.where().idEq(p.getId());
//				PreparedUpdate<Topic> pu = ub.prepare();
//				topicDao.update(pu);
//				
//				// DEBUG
////				p = topicDao.queryForId(p.getId());
////				Log.d(LOG_TAG, String.format("updated to %d.", p.getDownloaded_video_count()));
//				
//				p = p.getParentTopic();
//			}
			
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}
	
	public void denormalizeAncestry() {
		String fmt = "%s|%s";
		try {
			for (Video v : getVideoDao().queryForAll()) {
				String ancestry = "";
				Topic p = v.getParentTopic();
				while (p != null) {
					getTopicDao().refresh(p);
					ancestry = String.format(fmt, p.getId(), ancestry);
					p = p.getParentTopic();
				}
				if (ancestry.endsWith("|")) { // actually, this is true for all videos
					ancestry = ancestry.substring(0, ancestry.length()-1);
				}
				v.setAncestry(ancestry);
				getVideoDao().update(v);
			}
			for (Topic t : getTopicDao().queryForAll()) {
				String ancestry = "";
				Topic p = t.getParentTopic();
				while (p != null) {
					getTopicDao().refresh(p);
					ancestry = String.format(fmt, p.getId(), ancestry);
					p = p.getParentTopic();
				}
				if (ancestry.endsWith("|")) { // true for all but the root topic
					ancestry = ancestry.substring(0, ancestry.length()-1);
				}
				t.setAncestry(ancestry);
				getTopicDao().update(t);
			}
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}
	
	private List<String> getAllDownloadedYoutubeIds() {
		// youtube ids look like this: -h_x8TwC1ik
		String youtubeIdPattern = "^([-_a-zA-Z0-9]{11})(\\.\\d{2,4})?";
		Pattern pattern = Pattern.compile(youtubeIdPattern);
		Matcher matcher;
		
		File file = context.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS);
		List<String> allFilenames = Arrays.asList(file.list());
		List<String> result = new ArrayList<String>();
		for (String s : allFilenames) {
			matcher = pattern.matcher(s);
			if (matcher.find()) {
				result.add(matcher.group(1));
			}
		}
		return result;
		
	}
	
} @JsonIgnoreProperties(ignoreUnknown=true)
@DatabaseTable
public class Topic extends EntityBase implements Comparable<Topic>, Serializable {
	public static final String LOG_TAG = Topic.class.getSimpleName();
/*
 *     Full Topic 
 *     
    
    // Always in both
    "kind": "Topic",
    "title": "The Root of All Knowledge",
    
    // Full version of both
    "description": "All concepts fit into the root of all knowledge",
    "relative_url": "/#root",
    "ka_url": "http://www.khanacademy.org/#root",
    "backup_timestamp": "2012-10-16T22:05:27Z",
    
    // Topic only
    "community_questions_title": null,
    "community_questions_url": null,
    "tags": [],
    "init_custom_stack": null,
    "id": "root",
    "assessment_progress_key": "awOEn1j7Rz0sCb-JrZu5agXkun6N382jJwX5fWYEo",
    "topic_page_url": "/",
    "hide": true,
    "extended_slug": "",
    "standalone_title": "The Root of All Knowledge",
    "children": [

 */
	
	/*
	 *    Topic as child
	 *    
	 
	        "kind": "Topic",
            "hide": false,
            "title": "New and Noteworthy",
            "url": "http://www.khanacademy.org/#new-and-noteworthy",
            "key_id": null,
            "id": "new-and-noteworthy"

	 */
	
	private static final long serialVersionUID = -4876433391343270374L;
	
	public static final String CHILD_KIND_VIDEO = "Video";
	public static final String CHILD_KIND_TOPIC = "Topic";
	public static final String CHILD_KIND_NONE = "CHILD_KIND_NONE";
	public static final String CHILD_KIND_UNKNOWN = "CHILD_KIND_???";

	// columnName="_id" for CursorAdapter use.
	@DatabaseField(id=true, columnName="_id")
	String id;
	
	@DatabaseField
	String child_kind;
	
	@DatabaseField
	int video_count;
	
	@DatabaseField
	int downloaded_video_count;
	
	/** Youtube Id of the video whose thumbnail we will display with this topic. 
	 * This will be the first video descendant of this topic that has a thumbnail url). */
	@DatabaseField
	String thumb_id;
	
	@DatabaseField
    String standalone_title;
	
    Collection<EntityBase> children;
    
    @ForeignCollectionField(eager=false)
    Collection<Topic> childTopics;
    
    @ForeignCollectionField(eager=false)
    Collection<Video> childVideos;
    
	/**
	 * @return the id
	 */
	@Override
	public String getId() {
		return id;
	}
	/**
	 * @param id the id to set
	 */
	public void setId(String id) {
		this.id = id;
	}
	public String getChild_kind() {
		return child_kind;
	}
	public void setChild_kind(String child_kind) {
		this.child_kind = child_kind;
	}
	@Override
	public int getVideo_count() {
		return video_count;
	}
	public void setVideo_count(int video_count) {
		this.video_count = video_count;
	}
	/**
	 * @return the downloaded_video_count
	 */
	@Override
	public int getDownloaded_video_count() {
		return downloaded_video_count;
	}
	/**
	 * @param downloaded_video_count the downloaded_video_count to set
	 */
	public void setDownloaded_video_count(int downloaded_video_count) {
		this.downloaded_video_count = downloaded_video_count;
	}
	public String getThumb_id() {
		return thumb_id;
	}
	public void setThumb_id(String thumb_id) {
		this.thumb_id = thumb_id;
	}
	/**
	 * @return the standalone_title
	 */
	public String getStandalone_title() {
		return standalone_title;
	}
	/**
	 * @param standalone_title the standalone_title to set
	 */
	public void setStandalone_title(String standalone_title) {
		this.standalone_title = standalone_title;
	}
	
	public Collection<? extends EntityBase> getChildren() {
		Log.d(LOG_TAG, String.format("getChildren: %s", getId()));
		
		if (children != null && children.size() > 0) {
			// This must have been set on this object by Jackson.
			return children;
		}
		
		Collection<Video> childVideos = getChildVideos();
		Log.d(LOG_TAG, String.format("  childVideos is %s", childVideos == null ? "null" : String.valueOf(childVideos.size())));
		if (childVideos != null && childVideos.size() > 0) {
			return childVideos;
		}
		Collection<Topic> childTopics = getChildTopics();
		Log.d(LOG_TAG, String.format("  childTopics is %s", childTopics == null ? "null" : String.valueOf(childTopics.size())));
		if (childTopics != null && childTopics.size() > 0) {
			return childTopics;
		}
		return new ArrayList<EntityBase>();
	}
	public void setChildren(Collection<EntityBase> children) {
		this.children = children;
		for (EntityBase child : children) {
			child.setParentTopic(this);
		}
	}
	
	
	/**
	 * Implements Comparable<Topic>
	 */
	@Override
	public int compareTo(Topic other) {
		return getId().compareTo(other.getId());
	}
	
	@Override
	public boolean equals(Object other) {
		try {
			return ((Topic) other).getId().equals(getId());
		} catch (ClassCastException e) {
			return false;
		}
	}
	
	@Override
	public int hashCode() {
		return (getClass().hashCode() + getId().hashCode()) % Integer.MAX_VALUE;
	}
	
	@Override
	public BaseEntityUpdateVisitor<Topic> buildUpdateVisitor() {
		return new BaseEntityUpdateVisitor<Topic>(this) {
			@Override
			public void visit(Topic toUpdate) {
				super.visit(toUpdate);
				
				String value = getChild_kind();
				if (!isDefaultValue(value, String.class)) {
					toUpdate.setChild_kind(value);
				}
				
				value = getStandalone_title();
				if (!isDefaultValue(value, String.class)) {
					toUpdate.setStandalone_title(value);
				}
			}
		};
	}
	
	@Override
	public void accept(EntityVisitor visitor) {
		visitor.visit(this);
	}
	public Collection<Topic> getChildTopics() {
		return childTopics;
	}
	public Collection<Video> getChildVideos() {
		return childVideos;
	}
} @JsonIgnoreProperties(ignoreUnknown=true)
@DatabaseTable(tableName = "video")
public class Video extends EntityBase {
	
	public static final byte DL_STATUS_NOT_STARTED = 0;
	public static final byte DL_STATUS_IN_PROGRESS = 1;
	public static final byte DL_STATUS_COMPLETE    = 2;
	
	
	/*
	 *   Full Video
	 *   
	 
	 // Always in both
	"kind": "Video",
    "title": "CA Algebra I: Number Properties and Absolute Value",
    
     // In full versions of both
    "description": "1-7, number properties and absolute value equations",
    "relative_url": "/video/ca-algebra-i--number-properties-and-absolute-value",
    "ka_url": "http://www.khanacademy.org/video/ca-algebra-i--number-properties-and-absolute-value",
    "backup_timestamp": "2012-10-16T16:23:04Z",
	
	 // Video Only
    "has_questions": false,
    "extra_properties": null,
    "node_slug": "v/ca-algebra-i--number-properties-and-absolute-value",
    "url": "http://www.youtube.com/watch?v=ZouQdHSyelg&feature=youtube_gdata_player",
    "progress_key": "v24364",
    "author_names": [
        "Sal Khan"
    ],
    "duration": 722,
    "keywords": "californa, standards, algebra, inequality, equations",
    "youtube_id": "ZouQdHSyelg",
    "download_urls": {
        "mp4": "http://s3.amazonaws.com/KA-youtube-converted/ZouQdHSyelg.mp4/ZouQdHSyelg.mp4",
        "png": "http://s3.amazonaws.com/KA-youtube-converted/ZouQdHSyelg.mp4/ZouQdHSyelg.png",
        "m3u8": "http://s3.amazonaws.com/KA-youtube-converted/ZouQdHSyelg.m3u8/ZouQdHSyelg.m3u8"
    },
    "date_added": "2011-02-20T16:34:07Z",
    "views": 120729,
    "readable_id": "ca-algebra-i--number-properties-and-absolute-value"

	 */
	
	/*
	 *   Video as child
	 *   
	 *   
            "kind": "Video",
            "hide": false,
            "title": "CA Algebra I: Number Properties and Absolute Value",
            "url": "http://www.khanacademy.org/video/ca-algebra-i--number-properties-and-absolute-value",
            "key_id": 24364,
            "id": "ca-algebra-i--number-properties-and-absolute-value"
	 */

	// columnName="_id" for CursorAdapter use.
	@DatabaseField(generatedId=true)
	int _id;
	
	@DatabaseField
	String readable_id;
	
	@DatabaseField
	int download_status;
	
	@DatabaseField
	String keywords;
	
	@DatabaseField
	String progress_key;
	
	@DatabaseField
	int duration;
	
	@DatabaseField(index=true)
	String youtube_id;
	
	public static class DownloadUrls {
		String mp4;
		String png;
		String m3u8;
		/**
		 * @return the mp4
		 */
		public String getMp4() {
			return mp4;
		}
		/**
		 * @param mp4 the mp4 to set
		 */
		public void setMp4(String mp4) {
			this.mp4 = mp4;
		}
		/**
		 * @return the png
		 */
		public String getPng() {
			return png;
		}
		/**
		 * @param png the png to set
		 */
		public void setPng(String png) {
			this.png = png;
		}
		/**
		 * @return the m3u8
		 */
		public String getM3u8() {
			return m3u8;
		}
		/**
		 * @param m3u8 the m3u8 to set
		 */
		public void setM3u8(String m3u8) {
			this.m3u8 = m3u8;
		}
		
		
	}
	
	DownloadUrls download_urls;
	
	@DatabaseField
	String mp4url;
	
	@DatabaseField
	String pngurl;
	
	@DatabaseField
	String m3u8url;
	
	// datetime
	@DatabaseField
	String date_added;
	
	@DatabaseField
	int views;
	
	@DatabaseField
	long dlm_id;
	
	/**
	 * @return the download_status
	 */
	public int getDownload_status() {
		return download_status;
	}

	/**
	 * @param download_status the download_status to set
	 */
	public void setDownload_status(int download_status) {
		this.download_status = download_status;
	}

	/**
	 * @return the keywords
	 */
	public String getKeywords() {
		return keywords;
	}

	/**
	 * @param keywords the keywords to set
	 */
	public void setKeywords(String keywords) {
		this.keywords = keywords;
	}

	/**
	 * @return the progress_key
	 */
	public String getProgress_key() {
		return progress_key;
	}

	/**
	 * @param progress_key the progress_key to set
	 */
	public void setProgress_key(String progress_key) {
		this.progress_key = progress_key;
	}

	/**
	 * @return the duration
	 */
	public int getDuration() {
		return duration;
	}

	/**
	 * @param duration the duration to set
	 */
	public void setDuration(int duration) {
		this.duration = duration;
	}

	/**
	 * @return the youtube_id
	 */
	public String getYoutube_id() {
		return youtube_id;
	}

	/**
	 * @param youtube_id the youtube_id to set
	 */
	public void setYoutube_id(String youtube_id) {
		this.youtube_id = youtube_id;
	}

	/**
	 * @return the download_urls
	 */
	public DownloadUrls getDownload_urls() {
		if (download_urls == null) {
			download_urls = new DownloadUrls();
			download_urls.setM3u8(m3u8url);
			download_urls.setMp4(mp4url);
			download_urls.setPng(pngurl);
		}
		return download_urls;
	}

	/**
	 * @param download_urls the download_urls to set
	 */
	public void setDownload_urls(DownloadUrls download_urls) {
		this.download_urls = download_urls;
//		boolean n = download_urls == null;
//		this.setMp4url(n? null: download_urls.getMp4());
//		this.setM3u8url(n? null: download_urls.getM3u8());
//		this.setPngurl(n? null: download_urls.getPng());
	}

	/**
	 * @return the date_added
	 */
	public String getDate_added() {
		return date_added;
	}

	/**
	 * @param date_added the date_added to set
	 */
	public void setDate_added(String date_added) {
		this.date_added = date_added;
	}

	/**
	 * @return the views
	 */
	public int getViews() {
		return views;
	}

	/**
	 * @param views the views to set
	 */
	public void setViews(int views) {
		this.views = views;
	}

	public long getDlm_id() {
		return dlm_id;
	}

	public void setDlm_id(long dlm_id) {
		this.dlm_id = dlm_id;
	}

	/**
	 * @return the readable_id
	 */
	public String getReadable_id() {
		return readable_id;
	}

	/**
	 * @param readable_id the readable_id to set
	 */
	public void setReadable_id(String readable_id) {
		this.readable_id = readable_id;
	}

	/**
	 * @return the mp4url
	 */
	public String getMp4url() {
		return mp4url;
	}

	/**
	 * @return the pngurl
	 */
	public String getPngurl() {
		return pngurl;
	}

	/**
	 * @return the m3u8url
	 */
	public String getM3u8url() {
		return m3u8url;
	}

	/**
	 * @param mp4url the mp4url to set
	 */
	public void setMp4url(String mp4url) {
		this.mp4url = mp4url;
	}

	/**
	 * @param pngurl the pngurl to set
	 */
	public void setPngurl(String pngurl) {
		this.pngurl = pngurl;
	}

	/**
	 * @param m3u8url the m3u8url to set
	 */
	public void setM3u8url(String m3u8url) {
		this.m3u8url = m3u8url;
	}

	
	@Override
	public boolean equals(Object other) {
		try {
			return ((Video) other).getReadable_id().equals(getReadable_id());
		} catch (ClassCastException e) {
			return false;
		}
	}
	
	@Override
	public int hashCode() {
		return (getClass().hashCode() + getReadable_id().hashCode()) % Integer.MAX_VALUE;
	}

	/**
	 * Be sure to refresh this Video from the database before calling this, as it relies
	 * on a current value of download_staus.
	 */
	@Override
	public int getDownloaded_video_count() {
		return getDownload_status() == DL_STATUS_COMPLETE ? 1 : 0;
	}

	@Override
	public int getVideo_count() {
		return 1;
	}
	
	@Override
	public String getId() {
		return getReadable_id();
	}
	
	
	@Override
	public BaseEntityUpdateVisitor<Video> buildUpdateVisitor() {
		return new BaseEntityUpdateVisitor<Video>(this) {
			@Override
			public void visit(Video toUpdate) {
				super.visit(toUpdate);
				
				String value = Video.this.getDate_added();
				if (!isDefaultValue(value, String.class)) {
					toUpdate.setDate_added(value);
				}
				
				value = Video.this.getKeywords();
				if (!isDefaultValue(value, String.class)) {
					toUpdate.setKeywords(value);
				}
				
				value = Video.this.getProgress_key();
				if (!isDefaultValue(value, String.class)) {
					toUpdate.setProgress_key(value);
				}
				
				value = Video.this.getYoutube_id();
				if (!isDefaultValue(value, String.class)) {
					toUpdate.setYoutube_id(value);
				}
				
				DownloadUrls urls = Video.this.getDownload_urls();
				if (!isDefaultValue(urls, DownloadUrls.class)) {
					toUpdate.setDownload_urls(urls);
				}
				
				int n = Video.this.getDuration();
				if (!isDefaultValue(n, Integer.class)) {
					toUpdate.setDuration(n);
				}
				
				n = Video.this.getViews();
				if (!isDefaultValue(n, Integer.class)) {
					toUpdate.setViews(n);
				}
				
			}
			
			@Override
			protected boolean isDefaultValue(Object value, Class<?> valueType) {
				if (DownloadUrls.class.equals(valueType)) {
					return null == value;
				}
				return super.isDefaultValue(value, valueType);
			}
		};
	}
	
	@Override
	public void accept(EntityVisitor visitor) {
		visitor.visit(this);
	}
} public class KAAPIAdapter {
	
	public static final String LOG_TAG = KAAPIAdapter.class.getSimpleName();
	private final String consumerKey;
	private final String consumerSecret;
	private KADataService dataService;
	private User currentUser;
	
	public KAAPIAdapter(KADataService dataService, String consumerKey, String consumerSecret) {
		this.dataService = dataService;
		this.consumerKey = consumerKey;
		this.consumerSecret = consumerSecret;
	}
	
	public interface EntityCallback<T extends Object>  {
		public void call(T entity);
	}
	
	public interface UserLoginHandler {
		public void onUserLogin(User user);
	}
	
	/* ********************** User Update Listener ************************/
	private List<UserUpdateListener> listeners = new ArrayList<UserUpdateListener>();
	
	public interface UserUpdateListener {
		public void onUserUpdate(User user);
	}
	
	public void registerUserUpdateListener(UserUpdateListener l) {
		listeners.add(l);
	}
	
	public void unregisterUserUpdateListener(UserUpdateListener l) {
		listeners.remove(l);
	}
	
	private void doUserUpdate(User user) {
		for (UserUpdateListener l: listeners) {
			l.onUserUpdate(user);
		}
	}
	
	// Called by VideoProgressPostTask
	/*package*/ void doBadgeEarned(Badge badge) {
		Log.d(LOG_TAG, "doBadgeEarned");
		
		Intent intent = new Intent(ACTION_BADGE_EARNED);
		intent.putExtra(EXTRA_BADGE, badge);
		LocalBroadcastManager.getInstance(dataService).sendBroadcast(intent);
	}
	
	
	/**
	 * Call me from the UI thread, please.
	 * 
	 * Meant for use by broadcast receivers receiving ACTION_BADGE_EARNED.
	 * */
	public void toastBadge(Badge badge) {
		BadgeCategory category = badge.getCategory();
//		Dao<BadgeCategory, Integer> dao = dataService.getHelper().getDao(BadgeCategory.class);
//		dao.refresh(category);
		
		Toast toast = new Toast(dataService);
		View content = LayoutInflater.from(dataService).inflate(R.layout.badge, null, false);
		ImageView iconView = (ImageView) content.findViewById(R.id.badge_image);
		TextView pointsView = (TextView) content.findViewById(R.id.badge_points);
		TextView titleView = (TextView) content.findViewById(R.id.badge_title);
		TextView descView = (TextView) content.findViewById(R.id.badge_description);
		
		iconView.setImageResource(category.getIconResourceId());
		int points = badge.getPoints();
		if (points > 0) {
			pointsView.setText(points + "");
		} else {
			pointsView.setVisibility(View.GONE);
		}
		titleView.setText(badge.getDescription());
		descView.setText(badge.getSafe_extended_description());
		
		toast.setView(content);
		toast.setDuration(Toast.LENGTH_LONG);
		toast.setGravity(Gravity.TOP, 0, 200);
		toast.show();
	}
	
	public void testBadgeEarned() {
		Badge testBadge = new Badge();
		BadgeCategory testCategory = new BadgeCategory(1);
		testBadge.setPoints(500);
		testBadge.setDescription("Going Transonic");
		testBadge.setCategory(testCategory);
		testBadge.setSafe_extended_description("Quickly & correctly answer 10 exercise problems in a row (time limit depends on exercise difficulty)");
		doBadgeEarned(testBadge);
	}
	
	

	private String getCurrentUserId() {
		Log.d(LOG_TAG, "getCurrentUserId");
		String result = dataService.getSharedPreferences(Constants.SETTINGS_NAME, Context.MODE_PRIVATE)
				.getString(Constants.SETTING_USERID, null);
		Log.d(LOG_TAG, "    --> " + result == null ? "null" : result);
		return result;
	}
	
	/**
	 * Get the current user.
	 * 
	 * This will return the last user authenticated as long as they have not logged out. Specifically, when we launch
	 * the application, we do not explicitly check their credentials. Point totals, video progress, etc., will show
	 * until we make an explicit check (profile page or a video progress update).
	 * 
	 * Once we actually have to hit the api, if authentication fails, it will cause the current user to be set
	 * null and a user update will go out. At that point the entire app will behave as if the user is logged out.
	 * 
	 * @return the user or null.
	 */
	public User getCurrentUser() {
		if (currentUser == null) {
			// We may have just launched, and this isn't yet cached. Check for a non-null id in preferences.
			String userid = getCurrentUserId();
			if (userid == null) { // no user saved
				return null;
			}
		
			try {
				currentUser = dataService.getHelper().getUserDao().queryForId(userid);
			} catch (SQLException e) {
				// That's fine; pretend no user was logged in.
				e.printStackTrace();
			}
		}
		return currentUser;
	}
	
	
	public OAuthConsumer getConsumer(User user) {
		OAuthConsumer consumer = new CommonsHttpOAuthConsumer(consumerKey, consumerSecret);
		if (user != null) {
			String token = user.getToken();
			String secret = user.getSecret();
			if (token != null && secret != null) {
				consumer.setTokenWithSecret(token, secret);
			}
		}
		return consumer;
	}
	
	/**
	 * Set the current user id in shared preferences.  Usually, you'll want to call loginCurrentUser after this.
	 * 
	 * @param userid The user id to set.
	 */
	private void setCurrentUserId(String userid) {
		Log.d(LOG_TAG, "setCurrentUserId: " + userid);
		dataService.getSharedPreferences(Constants.SETTINGS_NAME, Context.MODE_PRIVATE)
				.edit()
				.putString(Constants.SETTING_USERID, userid)
				.apply();
	}
	
	private void setCurrentUser(User user) {
//		if (user == null) {
//			String url = "http://www.khanacademy.org/";
//			// TODO: Clear WebView cookies, so the "My Account" page is logged out as well.
//			// The idea is, use getCookie(url) and parse to get each cookie name, then setCookie with each "name=;"
//			// and possibly an expiration in the past.
//			CookieManager m = CookieManager.getInstance();
//			
//			// segfault (really?) on next line, unless CookieSyncManager.createInstance has been called.
//			String existing = m.getCookie(url);
//			// sample return value:
//			// fkey=1.0_FtAEhPDlC87EYQ==_1355510081; KAID="aHR0cDovL2dvb2dsZWlkLmtoYW5hY2FkZW15Lm9yZy8xMTgyMzkxMjIxOTM5MDEwNTQwNjUKMjAxMjM0OTE4MzQ1Ni45Mjk5MjAKODYyNGRmNzA5MGY1OGE2NzdiYjZlZTgxYTU4YjI4NmJmNTAwZTc2ZmNlNWE5MTkwYzNmYWZlNmY2MjMyMDVmMQ=="; gae_b_id=; GOOGAPPUID=23; return_visits_http%3A%2F%2Fgoogleid.khanacademy.org%2F118239122193901054065=1355510097.12996; __utma=3422703.2067802560.1355510099.1355510099.1355510099.1; __utmb=3422703.3.10.1355510099; __utmc=3422703; __utmz=3422703.1355510099.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmv=3422703.|1=User%20Type=Logged%20In=1^2=User%20Points=17300=1^3=User%20Videos=18=1^4=User%20Exercises=0=1
//			Log.d(LOG_TAG, existing);
//			
//			m.setCookie(url, "");
//			
//			// Naive solution, but this also clears facebook/google cookies, etc.
//			m.removeAllCookie();
//		}
		currentUser = user;
		setCurrentUserId(user == null ? null : user.getNickname());
	}

	
	public void login(final String token, final String secret, final UserLoginHandler handler) {
		OAuthConsumer consumer = new CommonsHttpOAuthConsumer(consumerKey, consumerSecret);
		consumer.setTokenWithSecret(token, secret);
		fetchUser(consumer, new EntityCallback<User>() {
			@Override
			public void call(User returnedUser) {
				
				if (returnedUser != null) {
					returnedUser.setToken(token);
					returnedUser.setSecret(secret);
					try {
						Dao<User, String> userDao = dataService.getHelper().getUserDao();
						if (userDao.getConnectionSource().isOpen()) {
							userDao.createOrUpdate(returnedUser);
						}
					} catch (SQLException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
					
				}
				handler.onUserLogin(returnedUser);
				currentUser = returnedUser;
				setCurrentUser(returnedUser);
				doUserUpdate(returnedUser);
			}
		});
	}
	
	/**
	 * Log the current user out.
	 * 
	 * @return true if a user was logged in, false otherwise.
	 */
	public boolean logout() {
		boolean result = this.isAuthenticated();
		setCurrentUser(null);
		doUserUpdate(null);
		Toast.makeText(dataService, dataService.getString(R.string.msg_logged_out),
				Toast.LENGTH_SHORT).show();
		return result;
	}
	
	/**
	 * Do we have an authenticated user registered?
	 * 
	 * Does not perform a request to validate the authentication; just returns
	 * true if we have what looks to be a valid consumer.
	 * 
	 * @return true if a user is logged in, false otherwise.
	 */
	public boolean isAuthenticated() {
		return this.currentUser != null;
	}
	
//  /api/v1/user
	/**
	 * Fetch data for the currently logged-in user. Requires prior authentication.
	 * 
	 * @param callback
	 */
	public void fetchUser(final OAuthConsumer consumer, final EntityCallback<User> callback) {
		String url = "http://www.khanacademy.org/api/v1/user";
		new KAEntityFetcherTask<User>(url, consumer) {
			@Override
			protected void onPostExecute(User result) {
				callback.call(result);
			}
		}.execute();
	}
	
	
	public void requestUserVideoUpdate(final User user) {
		fetchUserVideos(getConsumer(user), new EntityCallback<List<UserVideo>>() {
			@Override
			public void call(List<UserVideo> userVideos) {
				int numChanged = 0;
				if (userVideos == null) {
					// TODO
				}
				
				Dao<UserVideo, Integer> userVideoDao = null;
				try {
					userVideoDao = dataService.getHelper().getUserVideoDao();
				} catch (SQLException e) {
					// Without a Dao we won't be able to do any updates, but probably shouldn't crash just yet.
					e.printStackTrace();
					return;
				}
				
				Map<String, Object> values = new HashMap<String, Object>();
				for (UserVideo v : userVideos) {
					// TODO : is it important to remove old userVideos that don't appear in this response?
					values.put("user_id", v.getUser().getNickname());
					values.put("video_id", v.getVideo_id());
					try {
						// createOrUpdate would be slick here, but since ORMLite does not support
						// multi-valued primary keys we can't key on User + Video
						List<UserVideo> existing = userVideoDao.queryForFieldValues(values);
						if (existing.size() > 0) {
							UserVideo it = existing.get(0);
							// the newly downloaded video v is most current.
							v.setId(it.getId());
							userVideoDao.update(v);
						} else {
							userVideoDao.create(v);
						}
						numChanged++;
					} catch (SQLException e) {
						// Can't update this video, but no need to crash.
						e.printStackTrace();
					} catch (IllegalStateException e) {
						// Can occur when a video update is coming back while we shut down the app, if the db is already closed.
						// TODO : the real fix is to use startService for this task.
						e.printStackTrace();
					}
				}
				
				if (numChanged > 0) {
					// This will trigger the video fragments to refresh the UserVideo related to this User and their current video.
					doUserUpdate(user);
				}
			}
		});
	}
	
//  /api/v1/user/videos
	public void fetchUserVideos(final OAuthConsumer consumer, final EntityCallback<List<UserVideo>> callback) {
		String url = "http://www.khanacademy.org/api/v1/user/videos";
		new KAEntityCollectionFetcherTask<UserVideo>(UserVideo.class, url, consumer) {
			@Override
			protected void onPostExecute(List<UserVideo> result) {
				if (result == null) {
					exception.printStackTrace();
					// But don't crash.
				}
				callback.call(result);
			}
		}.execute();
	}
	
	public void postVideoProgress(final UserVideo userVideo, final Runnable successHandler, final Runnable errorHandler) {
		Log.d(LOG_TAG, "postVideoProgress");
		
		new VideoProgressPostTask(dataService) {
			@Override
			protected void onPostExecute(User returnedUser) {
				// if returnedUser is null, the post failed.
				if (returnedUser != null) {
					doUserUpdate(returnedUser);
					if (successHandler != null) {
						successHandler.run();
					}
				} else {
					if (errorHandler != null) {
						errorHandler.run();
					}
				}
			}
		}.execute(userVideo);
	}
	
} public class LibraryUpdaterTask extends AsyncTask<Void, Void, Integer> {
	
	public static final String LOG_TAG = LibraryUpdaterTask.class.getSimpleName();
	
	private final OrmLiteSqliteOpenHelper tempDbHelper;
	private final String topicTableName = "topic";
	private final String videoTableName = "video";
	private final String topicvideoTableName = "topicvideo";
	private final KADataService dataService;
	private final ConnectivityManager connectivityManager;
	
	private final String url = "http://www.khanacademy.org/api/v1/topictree";
	public boolean force = false;
	
	private static final List<String> downloadUrlFields = Arrays.asList(new String[] {"png", "mp4", "m3u8"});
	private static final List<String> stringFields = Arrays.asList(new String[] {"kind", "title", "description", "ka_url", "id", "readable_id", "youtube_id", "keywords", "progress_key", "date_added"}); // 10
	private static final List<String> booleanFields = Arrays.asList(new String[] {"hide"});
	private static final List<String> intFields =  Arrays.asList(new String[] {"views", "duration"});
	// "children", "download_urls" are special

	/** The return values we need when parsing an array of children. */
	private class ChildArrayResults {
		List<String> childIds = new ArrayList<String>();
		String childKind;
		int videoCount;
		String thumbId;
	}
	
	public LibraryUpdaterTask(KADataService dataService) {
		this.dataService = dataService;
		this.tempDbHelper = new TempHelper(dataService, dataService.getHelper().getReadableDatabase());
		connectivityManager = (ConnectivityManager) dataService.getSystemService(Context.CONNECTIVITY_SERVICE);
	}
	
	@Override
	protected Integer doInBackground(Void... params) {
		
		SharedPreferences prefs = dataService.getSharedPreferences(SETTINGS_NAME, Context.MODE_PRIVATE);
		
		// Connectivity receiver.
		final NetworkInfo activeNetwork = connectivityManager.getActiveNetworkInfo();
		ComponentName receiver = new ComponentName(dataService, WifiReceiver.class);
		PackageManager pm = dataService.getPackageManager();
		if (activeNetwork == null || !activeNetwork.isConnected()) {
			// We've missed a scheduled update. Enable the receiver so it can launch an update when we reconnect.
			Log.d(LOG_TAG, "Missed library update: not connected. Enabling connectivity receiver.");
			pm.setComponentEnabledSetting(receiver,
			        PackageManager.COMPONENT_ENABLED_STATE_ENABLED,
			        PackageManager.DONT_KILL_APP);
			return RESULT_CODE_FAILURE;
		} else {
			// We are connected. Disable the receiver.
			Log.d(LOG_TAG, "Library updater connected. Disabling connectivity receiver.");
			pm.setComponentEnabledSetting(receiver,
			        PackageManager.COMPONENT_ENABLED_STATE_DISABLED,
			        PackageManager.DONT_KILL_APP);
		}
		
		InputStream in = null;
		String etag = prefs.getString(SETTING_LIBRARY_ETAG, null);
		
		try {
			HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
			if (etag != null && !force) {
				conn.setRequestProperty("If-None-Match", etag);
			}
			
			int code = conn.getResponseCode();
			switch (code) {
			case HttpStatus.SC_NOT_MODIFIED:
				// If we got a 304, we're done.
				// Use failure code to indicate there is no temp db to copy over.
				Log.d(LOG_TAG, "304 in library response.");
				return RESULT_CODE_FAILURE;
			default:
				// Odd, but on 1/3/13 I received correct json responses with a -1 for responseCode. Fall through.
				Log.w(LOG_TAG, "Error code in library response: " + code);
			case HttpStatus.SC_OK:
				// Parse response.
				in = conn.getInputStream();
				JsonFactory factory = new JsonFactory();
				final JsonParser parser = factory.createJsonParser(in);
				
				SQLiteDatabase tempDb = tempDbHelper.getWritableDatabase();
				tempDb.beginTransaction();
				try {
					tempDb.execSQL("delete from topic");
					tempDb.execSQL("delete from topicvideo");
					tempDb.execSQL("delete from video");
					
					parseObject(parser, tempDb, null, 0);
					tempDb.setTransactionSuccessful();
				} catch (Exception e) {
					e.printStackTrace();
					return RESULT_CODE_FAILURE;
				} finally {
					tempDb.endTransaction();
					tempDb.close();
				}
				
				// Save etag once we've successfully parsed the response.
				etag = conn.getHeaderField("ETag");
				prefs.edit().putString(SETTING_LIBRARY_ETAG, etag).apply();
				
				// Move this new content from the temp db into the main one.
				mergeDbs();
				
				return RESULT_CODE_SUCCESS;
			}
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (JsonParseException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			tempDbHelper.close();
		}
		
		return RESULT_CODE_FAILURE;
	}

	private ContentValues parseObject(JsonParser parser, SQLiteDatabase tempDb, String parentId, int seq) throws JsonParseException, IOException {
		// TODO : Grab id of root topic here, and store it in shared prefs, in case it ever
		//        changes. Currently we assume "root" and a change would be catastrophic.
		ContentValues result = new ContentValues();
		ChildArrayResults childResults = null;
		boolean badKind = false;
		
		result.put("parentTopic_id", parentId);
		result.put("seq", seq);
		
		while (parser.nextValue() != JsonToken.END_OBJECT) {
			
			// Allows us to burn through the rest of the object once we discover it's an exercise or something else we don't care about.
			if (badKind) continue;
			
			String fieldName = parser.getCurrentName();
			
			// Keys present will determine object type.
			if (stringFields.contains(fieldName)) {
				// Use getValueAsString over getText; getText returns "null" while getValueAsString returns null.
				String value = parser.getValueAsString();
				result.put(fieldName, value);
				
				if ("id".equals(fieldName)) {
					if (childResults != null) {
						addParentIdToChildren(tempDb, childResults, value);
					}
				}
			} else if (intFields.contains(fieldName)) {
				result.put(fieldName, parser.getIntValue());
			} else if (booleanFields.contains(fieldName)) {
				result.put(fieldName, parser.getBooleanValue());
			} else if ("children".equals(fieldName)) {
				childResults = parseChildArray(parser, tempDb, result.containsKey("id") ? result.getAsString("id") : null);
				result.put("video_count", childResults.videoCount);
				result.put("child_kind", childResults.childKind);
				result.put("thumb_id", childResults.thumbId);
			} else if ("download_urls".equals(fieldName)) {
				parseDownloadUrls(parser, result);
			} else if (null == fieldName) {
				// Noop. Just in case.
			} else {
				JsonToken next = parser.getCurrentToken();
				if (next == JsonToken.START_OBJECT || next == JsonToken.START_ARRAY) {
					// Skip this object or array, leaving us pointing at the matching end_object / end_array token.
					parser.skipChildren();
				}
			}
		}
		
		// Ignore types we don't need.
		if (badKind) {
			return null;
		}
		
		// Having parsed this whole object, we can insert it.
		if (result.containsKey("kind")) {
			String kind = result.getAsString("kind");
			if ("Topic".equals(kind)) {
				if (result.containsKey("id")) {
					result.put("_id", result.getAsString("id"));
					result.remove("id");
				}
				if (result.containsKey("child_kind")) {
					String child_kind = result.getAsString("child_kind");
					if ("Topic".equals(child_kind) || "Video".equals(child_kind)) {
						insertTopic(tempDb, result);
					}
				}
			} else if ("Video".equals(kind)) {
				if (result.containsKey("id")) {
					result.put("video_id", result.getAsString("id"));
					result.remove("id");
				}
				insertTopicVideo(tempDb, result);
				insertVideo(tempDb, result);
			}
		}
		
		return result;
	}
	
	// return list of ids of the children in this array, so the parent can attach a parentTopic_id after the fact.
	private ChildArrayResults parseChildArray(JsonParser parser, SQLiteDatabase tempDb, String parentTopic_id) throws JsonParseException, IOException {
		ChildArrayResults result = new ChildArrayResults();
		int seq = 0;
		
		JsonToken currentToken = parser.getCurrentToken();
		if (currentToken == JsonToken.START_ARRAY) {
			while (parser.nextValue() == JsonToken.START_OBJECT) { // Otherwise, we will be at END_ARRAY here.
				ContentValues values = parseObject(parser, tempDb, parentTopic_id, seq++);
				
				if (values != null && values.containsKey("kind")) {
					String kind = values.getAsString("kind");
					
					if ("Topic".equals(kind) && values.containsKey("_id")) {
						result.childKind = kind;
						result.childIds.add(values.getAsString("_id"));
						result.videoCount += values.getAsInteger("video_count");
						if (result.thumbId == null && values.containsKey("thumb_id")) {
							// Return the first available thumb id as this topic's thumb id.
							result.thumbId = values.getAsString("thumb_id");
						}
					} else if ("Video".equals(kind) && values.containsKey("readable_id")) {
						result.childKind = kind;
						result.childIds.add(values.getAsString("readable_id"));
						result.videoCount += 1;
						if (result.thumbId == null && values.containsKey("pngurl")) {
							// Return youtube_id of first video with a thumbnail as this topic's thumbnail id.
							result.thumbId = values.getAsString("youtube_id");
						}
					}
				}
			}
		}
		return result;
	}
	
	private void parseDownloadUrls(JsonParser parser, ContentValues result) throws JsonParseException, IOException {
		// parser points at begin object token right now
		if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
			while (parser.nextValue() != JsonToken.END_OBJECT) {
				String fieldName = parser.getCurrentName();
				
				if (downloadUrlFields.contains(fieldName)) {
					result.put(fieldName + "url", parser.getValueAsString());
				}
			}
		}
//		else we must not have any download urls (null, '', something like that.)
	}
	
	// When we encounter our id, if we have already parsed children, then we must select all of 
	// the correct type, with the ids we received from the children, who don't have parentTopic_ids, and set them.
	// It appears, looking at the tree as returned 12/30/12, that this is unnecessary. It's here just in case.
	private void addParentIdToChildren(SQLiteDatabase tempDb, ChildArrayResults childResults, String parentId) {
		
		if (childResults == null || childResults.childIds == null || childResults.childIds.size() == 0) {
			return;
		}
		
		ContentValues values = new ContentValues();
		values.put("parentTopic_id", parentId);
		
		boolean isTopic = "Topic".equals(childResults.childKind);
		String table = isTopic ? topicTableName : videoTableName;
		
		StringBuilder where = new StringBuilder();
		where.append(isTopic ? "_id IN (?" : "readable_id IN (?");
		// First '?' already added, so start at 1.
		for (int i = 1; i < childResults.childIds.size(); ++i) {
			where.append(",?");
		}
		where.append(")");
		String whereClause = where.toString();
		
		String[] whereArg = childResults.childIds.toArray(new String[childResults.childIds.size()]);
		
		tempDb.update(table, values, whereClause, whereArg);
	}
	
	private void insertTopic(SQLiteDatabase tempDb, ContentValues values) {
		tempDb.insert(topicTableName, null, values);
	}
	
	private void insertTopicVideo(SQLiteDatabase tempDb, ContentValues values) {
		// This values is for a video.
		ContentValues v = new ContentValues();
		v.put("topic_id", values.getAsString("parentTopic_id"));
		v.put("video_id", values.getAsString("readable_id"));
		tempDb.insertWithOnConflict(topicvideoTableName, null, v, SQLiteDatabase.CONFLICT_IGNORE);
	}
	
	private void insertVideo(SQLiteDatabase tempDb, ContentValues values) {
		tempDb.insertWithOnConflict(videoTableName, null, values, SQLiteDatabase.CONFLICT_IGNORE);
	}
	
	private void mergeDbs() {
		Log.d(LOG_TAG, "update received - juggling dbs");
		// Get main database, attach temp db to it.
		SQLiteDatabase mainDb = dataService.getHelper().getWritableDatabase();
		mainDb.execSQL("attach database ? as ka_temp", new Object[] {dataService.getDatabasePath("ka_temp").getAbsolutePath()});
		
		mainDb.beginTransaction();
		try {
			
			// Maintain download status.
			String sql = "select max(download_status), dlm_id, youtube_id from video where download_status != ? group by youtube_id";
			Cursor c = mainDb.rawQuery(sql, new String[] {"" + Video.DL_STATUS_NOT_STARTED});
			Cursor c1;
			String[] videoIds = new String[c.getCount()];
			int i = 0;
			while (c.moveToNext()) {
				String youtube_id = c.getString(c.getColumnIndex("youtube_id"));
				String download_status = c.getString(c.getColumnIndex("max(download_status)"));
				long dlm_id = c.getLong(c.getColumnIndex("dlm_id"));
				videoIds[i++] = youtube_id;
				ContentValues v = new ContentValues();
				v.put("download_status", download_status);
				v.put("dlm_id", dlm_id);
				String[] idArg = new String[] {youtube_id};
				mainDb.update("ka_temp.video", v, "youtube_id = ?", idArg);
				
				// cursor over parent topics of this video
				sql = "select ka_temp.topic._id from ka_temp.topic, ka_temp.topicvideo, ka_temp.video where ka_temp.video.youtube_id=? and ka_temp.topicvideo.video_id=ka_temp.video.readable_id and ka_temp.topicvideo.topic_id=ka_temp.topic._id";
				c1 = mainDb.rawQuery(sql, idArg);
				Log.d(LOG_TAG, String.format("updating counts for %d topics", c1.getCount()));
				while (c1.moveToNext()) {
					String topicId = c1.getString(c1.getColumnIndex("_id"));
					DatabaseHelper.incrementDownloadedVideoCounts(mainDb, topicId, "ka_temp.topic");
				}
				c1.close();
			}
			c.close();
			
			mainDb.execSQL("delete from topic");
			mainDb.execSQL("insert into topic select * from ka_temp.topic");
			
			mainDb.execSQL("delete from topicvideo");
			mainDb.execSQL("insert into topicvideo select * from ka_temp.topicvideo");
		
			mainDb.execSQL("delete from video");
			mainDb.execSQL("insert into video select * from ka_temp.video");
			
			mainDb.setTransactionSuccessful();
		} finally {
			mainDb.endTransaction();
			mainDb.execSQL("detach database ka_temp");
		}
		
		Log.d(LOG_TAG, "finished juggling");
	}
	
	private class TempHelper extends OrmLiteSqliteOpenHelper {
		
		static final String DB_NAME = "ka_temp";
		final List<String> schema = new ArrayList<String>();
		
		TempHelper(Context context, SQLiteDatabase mainDb) {
			super(context, DB_NAME, null, 1, R.raw.ormlite_config);
			
			context.deleteDatabase(DB_NAME);
			
			Cursor cursor = mainDb.rawQuery("select sql from sqlite_master where tbl_name=?", new String[] {"topic"});
			while (cursor.moveToNext()) {
				String sql = cursor.getString(0);
				Log.d(LOG_TAG, "TempHelper: " + sql);
				if (sql != null) {
					schema.add(sql);
				}
			}
			cursor.close();
			
			cursor = mainDb.rawQuery("select sql from sqlite_master where tbl_name=?", new String[] {"video"});
			while (cursor.moveToNext()) {
				String sql = cursor.getString(0);
				Log.d(LOG_TAG, "TempHelper: " + sql);
				if (sql != null) {
					schema.add(sql);
				}
			}
			cursor.close();
			
			cursor = mainDb.rawQuery("select sql from sqlite_master where tbl_name=?", new String[] {"topicvideo"});
			while (cursor.moveToNext()) {
				String sql = cursor.getString(0);
				Log.d(LOG_TAG, "TempHelper: " + sql);
				if (sql != null) {
					schema.add(sql);
				}
			}
			cursor.close();
		}

		@Override
		public void onCreate(SQLiteDatabase database,
				ConnectionSource connectionSource) {
			
			Log.d(LOG_TAG, "TempHelper.onCreate");
			for (String sql : schema) {
				Log.d(LOG_TAG, sql);
				database.execSQL(sql);
			}
			
		}

		@Override
		public void onUpgrade(SQLiteDatabase database,
				ConnectionSource connectionSource, int oldVersion,
				int newVersion) {
		}
	}
	
} public class CaptionManager {
	/* ***********************   STATIC   ***************************/
	
	public static final String LOG_TAG = CaptionManager.class.getSimpleName();

	public static final int CONNECT_TIMEOUT = 5000;
	
	/* ***********************   PRIVATE   ***************************/
	
	private KADataService dataService;
	private String subtitleFormat;
	
	/* ***********************   PUBLIC   ***************************/

	/**
	 * Get a CaptionManager to manage your captions!
	 * 
	 * @param context Any context will do. Used to look up a string resource, and no reference is kept.
	 */
	public CaptionManager(KADataService dataService) {
		this.dataService = dataService;
		subtitleFormat = dataService.getString(R.string.url_format_subtitles);
	}
	
	/**
	 * Get a {@link WebResourceResponse} with subtitles for the video with the given youtube id.
	 * 
	 * The response contains a UTF-8 encoded json object with the subtitles received 
	 * from universalsubtitles.org. 
	 * 
	 * @param youtubeId The youtube id of the video whose subtitles we need.
	 * @return The {@link WebResourceResponse} with the subtitles, or {@code null} in case of error or if none are found.
	 */
	public WebResourceResponse fetchRawCaptionResponse(String youtubeId) {
		Log.d(LOG_TAG, "fetchRawCaptionResponse");
		String youtube_url = "http://www.youtube.com/watch?v=" + youtubeId;
		try {
			URL url = new URL(String.format(subtitleFormat, youtube_url, "en"));
			URLConnection connection = url.openConnection();
			connection.setConnectTimeout(CONNECT_TIMEOUT);
			connection.setUseCaches(true);
			InputStream in = null;
			try {
				in = connection.getInputStream();
			} catch (SocketTimeoutException e) {
				e.printStackTrace();
			} catch (Exception e) {
				e.printStackTrace();
				//various exceptions including at least ConnectException and UnknownHostException can happen if we're offline
			}
			
			return in==null? null: new WebResourceResponse("application/json", "UTF-8", in);
		} catch (ClientProtocolException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}

		return null;
	}
	
	public List<Caption> getCaptions(String youtubeId) {
		Log.d(LOG_TAG, "getCaptions: " + youtubeId);
		
		List<Caption> result = null;
		Dao<Caption, Integer> captionDao = null;
		try {
			captionDao = dataService.getHelper().getDao(Caption.class);
			QueryBuilder<Caption, Integer> q = captionDao.queryBuilder();
			q.where().eq("youtube_id", youtubeId);
			q.orderBy("sub_order", true);
			// TODO : Avoid inserting duplicates in the first place, and do a migration to clean up.
			q.groupBy("sub_order");
			result = q.query();
		} catch (SQLException e) {
			e.printStackTrace();
		}
		if (result != null && result.size() > 0) {
			Log.d(LOG_TAG, " already cached; returning");
			return result;
		}
		
		// If we do not already have the captions, try fetching them.
		// Clients will call this in a background thread, so we can take our time.
		WebResourceResponse response = fetchRawCaptionResponse(youtubeId);
		result = parseAPIResponse(response);
		result = pruneEmptyCaptions(result);
		result = persist(result, youtubeId);
		return result;
	}
	
	private List<Caption> parseAPIResponse(WebResourceResponse response) {
		Log.d(LOG_TAG, "parseAPIResponse");
		List<Caption> result = null;
		if (response != null) {
			ObjectMapper mapper = new ObjectMapper();
			try {
				result = mapper.readValue(response.getData(), new TypeReference<List<Caption>>(){});
				Log.d(LOG_TAG, " result length is " + result.size());
			} catch (JsonParseException e) {
				// At 5pm on a Thursday, I encountered this exception. Did a little digging, and the response turned
				// out to contain this: 
				// <html> <body> <div style="text-align: center; padding-top: 200px">
				//  	Amara is currently unavailable for scheduled maintenance. The site will be back shortly.
				// </div> </body> </html>
				
				// Another at 2:45 Tuesday:  Illegal character ((CTRL-CHAR, code 31)): only regular white space (\r, \n, \t) is allowed between tokens
				// This has happened more than once around the same time. Caught it again today, Tue 12/4, at about 3:00.
				
				// At any rate, these all fall into the "failed to download" category rather than the "none exist" category.
				e.printStackTrace();
			} catch (JsonMappingException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		} else {
			Log.d(LOG_TAG, " response was null");
		}
		return result;
	}
	
	private List<Caption> pruneEmptyCaptions(List<Caption> toPrune) {
		Log.d(LOG_TAG, "pruneEmptyCaptions");
		// Ensure captions have times. See long comment below for explanation.
		// Otherwise, return null and pretend there were no captions.
		List<Caption> result = null;
		if (toPrune != null) {
			for (Caption c : toPrune) {
//				Log.d(LOG_TAG, String.format("%03.2f %10d %s", c.getSub_order(), c.getStart_time(), c.getText()));
				if (c.getStart_time() > 0) {
					result = toPrune;
					break;
				}
			}
		}
		
		// prune individual empty captions
		if (result != null && result.size() > 0) {
			List<Caption> toRemove = new ArrayList<Caption>();
			for (Caption caption : result) {
				if (caption.getText().trim().length() == 0) {
					toRemove.add(caption);
				}
			}
			for (Caption caption : toRemove) {
				result.remove(caption);
			}
		}
		return result;
	}
	
	private List<Caption> persist(final List<Caption> toSave, final String youtubeId) {
		try {
			final Dao<Caption, Integer> captionDao = dataService.getHelper().getDao(Caption.class);
			
			if (captionDao != null && toSave != null && toSave.size() > 0) {
				// Batching speeds this up significantly.
				captionDao.callBatchTasks(new Callable<Void>() {
					@Override
					public Void call() throws Exception {
						for (Caption c : toSave) {
							try {
								c.setYoutube_id(youtubeId);
								captionDao.create(c);
							} catch (SQLException e) {
								e.printStackTrace();
							}
						}
						return null;
					}
				});
			}
		} catch (SQLException e) {
			e.printStackTrace();
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		return toSave;
	}
	
		/*  Example of a response that isn't quite what we want.
		 *  In these cases, we just pretend there are no subtitles at all, as an incomplete list without time stamps isn't that useful.
		 * [
    {
        "subtitle_id": "wcpvewparc82371936",
        "text": "What I want to do in this video is show you that some of the things that we've been talking about ",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 1.0,
        "start_of_paragraph": false
    },
    {
        "subtitle_id": "ivddkbpjit82695805",
        "text": "on the last few videos actually do happen, and in particular,",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 2.0,
        "start_of_paragraph": false
    },
    {
        "subtitle_id": "avfjvqzffm83421791",
        "text": "talk about how one of these speculative attacks on a currency can turn into",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 3.0,
        "start_of_paragraph": false
    },
    {
        "subtitle_id": "fmfcpugiut83483196",
        "text": "a banking crisis! This is a chart",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 4.0,
        "start_of_paragraph": false
    },
    {
        "subtitle_id": "otacwxlpyh83540836",
        "text": "from Oxford Economics, and it shows two things:",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 5.0,
        "start_of_paragraph": false
    },
    {
        "subtitle_id": "yyqivgduxz83628118",
        "text": "Thailand's exchange rate and short-term interest rates from the early",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 6.0,
        "start_of_paragraph": false
    },
    {
        "subtitle_id": "rtbbbazapm83760974",
        "text": "1990's to the present, there's a couple of interesting",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 7.0,
        "start_of_paragraph": false
    },
    {
        "subtitle_id": "ccxmomlhbe83829991",
        "text": "things that you might see over here; the first is the exchange rate, you see",
        "start_time": -1,
        "end_time": -1,
        "sub_order": 8.0,
        "start_of_paragraph": false
    }
]
		 */
} public class Log {
	
	public static final String LOG_TAG = "KA";
	
	public static final int VERBOSE = 0;
	public static final int DEBUG = 1;
	public static final int INFO = 2;
	public static final int WARN = 3;
	public static final int ERROR = 4;
	public static final int ASSERT = 5;
	public static final int SUPPRESS = 6;
	
	private static Map<String, Integer> enabledTags = new HashMap<String, Integer> ();
	private static int defaultLogLevel = SUPPRESS;
	
	/**
	 * Set the default logging level.  If no level has been set for a given tag, this
	 * default level will determine whether the message is logged. 
	 * 
	 * @param defaultLevel The default log level.
	 */
	public static void setDefaultLevel(int defaultLevel) {
		defaultLogLevel = defaultLevel;
	}
	
	/**
	 * Get the default log level, which is used to determine whether a message is logged
	 * if no level has been explicitly set for its tag.
	 * 
	 * @return The default log level.
	 */
	public static int getDefaultLevel() {
		return defaultLogLevel;
	}
	
	/**
	 * Get the log level for a given tag.
	 * 
	 * @param tag The tag for which to return a log level.
	 * @return The minimum level at which logs will be posted for the given tag.
	 */
	public static int getLevel(String tag) {
		Integer level = enabledTags.get(tag);
		if (level == null) return SUPPRESS;
		return level;
	}
	
	/**
	 * Enable a tag for future log messages.
	 * 
	 * @param tag The tag to enable, or "*" to set the default.
	 * @param level The minimum level to log.
	 */
	public static void setLevel(String tag, int level) {
		enabledTags.put(tag, level);
	}
	
	/**
	 * Disable a tag for future log messages.
	 * 
	 * @param tag The tag to disable, or "*" to suppress logging by default.
	 */
	public static void disable(String tag) { 
		enabledTags.remove(tag);
	}
	
	/**
	 * Forward a message to the android logger.  
	 * 
	 * The message will be forwarded only if its tag has previously been enabled at the 
	 * DEBUG or lower level, or if the default log level is DEBUG or lower. 
	 * 
	 * The tag is wrapped into the message, and sent to the android 
	 * logger with the application's tag.  For a call like Log.d("a", "b") this results in 
	 * the call android.util.Log.d("KA", "a: b") if "a" is enabled at DEBUG or lower, and 
	 * no operation otherwise.
	 * 
	 * @param tag The tag to check against the enabled list and prepend to the message.
	 * @param msg The message to log.
	 */
	public static void d(String tag, String msg) {
		Integer level = enabledTags.get(tag);
		if ( (level != null && level <= DEBUG) || (level == null && defaultLogLevel <= DEBUG) ) {
			android.util.Log.d(LOG_TAG, tag + ": " + msg);
		}
	}
	
	/**
	 * Forward a message to the android logger.  
	 * 
	 * The message will be forwarded only if its tag has previously been enabled at the 
	 * ERROR or lower level, or if the default log level is ERROR or lower. 
	 * 
	 * The tag is wrapped into the message, and sent to the android 
	 * logger with the application's tag.  For a call like Log.e("a", "b") this results in 
	 * the call android.util.Log.e("KA", "a: b") if "a" is enabled at ERROR or lower, and 
	 * no operation otherwise.
	 * 
	 * @param tag The tag to check against the enabled list and prepend to the message.
	 * @param msg The message to log.
	 */
	public static void e(String tag, String msg) {
		Integer level = enabledTags.get(tag);
		if ( (level != null && level <= ERROR) || (level == null && defaultLogLevel <= ERROR) ) {
			android.util.Log.e(LOG_TAG, tag + ": " + msg);
		}
	}
	
	/**
	 * Forward a message to the android logger.  
	 * 
	 * The message will be forwarded only if its tag has previously been enabled at the 
	 * WARN or lower level, or if the default log level is WARN or lower. 
	 * 
	 * The tag is wrapped into the message, and sent to the android 
	 * logger with the application's tag.  For a call like Log.w("a", "b") this results in 
	 * the call android.util.Log.w("KA", "a: b") if "a" is enabled at WARN or lower, and 
	 * no operation otherwise.
	 * 
	 * @param tag The tag to check against the enabled list and prepend to the message.
	 * @param msg The message to log.
	 */
	public static void w(String tag, String msg) {
		Integer level = enabledTags.get(tag);
		if ( (level != null && level <= WARN) || (level == null && defaultLogLevel <= WARN) ) {
			android.util.Log.w(LOG_TAG, tag + ": " + msg);
		}
	}
	
	/**
	 * Forward a message to the android logger.  
	 * 
	 * The message will be forwarded only if its tag has previously been enabled at the 
	 * INFO or lower level, or if the default log level is INFO or lower. 
	 * 
	 * The tag is wrapped into the message, and sent to the android 
	 * logger with the application's tag.  For a call like Log.i("a", "b") this results in 
	 * the call android.util.Log.i("KA", "a: b") if "a" is enabled at INFO or lower, and 
	 * no operation otherwise.
	 * 
	 * @param tag The tag to check against the enabled list and prepend to the message.
	 * @param msg The message to log.
	 */
	public static void i(String tag, String msg) {
		Integer level = enabledTags.get(tag);
		if ( (level != null && level <= INFO) || (level == null && defaultLogLevel <= INFO) ) {
			android.util.Log.i(LOG_TAG, tag + ": " + msg);
		}
	}
	
	/**
	 * Forward a message to the android logger.  
	 * 
	 * The message will be forwarded only if its tag has previously been enabled at the 
	 * VERBOSE level, or if the default log level is VERBOSE. 
	 * 
	 * The tag is wrapped into the message, and sent to the android 
	 * logger with the application's tag.  For a call like Log.v("a", "b") this results in 
	 * the call android.util.Log.v("KA", "a: b") if "a" is enabled at VERBOSE, and 
	 * no operation otherwise.
	 * 
	 * @param tag The tag to check against the enabled list and prepend to the message.
	 * @param msg The message to log.
	 */
	public static void v(String tag, String msg) {
		Integer level = enabledTags.get(tag);
		if ( (level != null && level <= VERBOSE) || (level == null && defaultLogLevel <= VERBOSE) ) {
			android.util.Log.v(LOG_TAG, tag + ": " + msg);
		}
	}
	
	/**
	 * Convenience function for passing in the level at runtime.
	 */
	public static void log(int priority, String tag, String msg) {
		switch (priority) {
		case VERBOSE:
			v(tag, msg); break;
		case INFO:
			i(tag, msg); break;
		case DEBUG:
			d(tag, msg); break;
		case WARN:
			w(tag, msg); break;
		case ERROR:
			e(tag, msg); break;
		case ASSERT:
		case SUPPRESS:
		default:
			// nop
		}
	
	}
	
} public interface ObjectCallback<T> {
	public void call(T obj);
} public class OfflineVideoManager {

	public static final String LOG_TAG = OfflineVideoManager.class.getSimpleName();
	
	private static final long MIN_ERROR_TOAST_INTERVAL = 500;
	private static final Pattern filenamePattern = Pattern.compile("([-_a-zA-Z0-9]{11})\\.mp4");

	
	/* ***********************  PRIVATE  ****************************/
	
	private final Handler handler = new Handler();
	private final ExecutorService pollerExecutor = Executors.newSingleThreadExecutor();
	private ExecutorService queueExecutor = Executors.newSingleThreadExecutor();
	
	private final KADataService dataService;
	private final Dao<Video, String> videoDao;
	private final LocalBroadcastManager broadcastManager;
	
	private volatile boolean shouldPoll = false;
	
	private Poller currentPoller;
	private FileObserver fileObserver;
	private long lastErrorToastTime;

	/**
	 * Poll the DownloadManager for progress of current downloads, then trigger a broadcast.
	 * 
	 * Not a particularly lengthy operation, but it happens a lot and we want it on a background thread for maximum smoothness.
	 */
	private class Poller extends AsyncTask<Void, Void, HashMap<String, Integer>> {
		
		private static final String sql = "select dlm_id from video where dlm_id > 0";
		
		private final DownloadManager.Query q = new DownloadManager.Query();

		@Override
		protected HashMap<String, Integer> doInBackground(Void... arg0) {
			// get an array of ids, for use in the download manager query
			Cursor c = dataService.getHelper().getReadableDatabase().rawQuery(sql, null);
			long[] ids = new long[c.getCount()];
			while (c.moveToNext()) {
				ids[c.getPosition()] = c.getLong(c.getColumnIndex("dlm_id"));
			}
			c.close();
		
			if (ids.length > 0) {
				q.setFilterById(ids);
				q.setFilterByStatus(DownloadManager.STATUS_RUNNING);
				
				Cursor cursor = getDownloadManager().query(q);
				HashMap<String, Integer> update = new HashMap<String, Integer>(cursor.getCount());
				while (cursor.moveToNext()) {
					String filename = cursor.getString(cursor.getColumnIndex(DownloadManager.COLUMN_LOCAL_FILENAME));
					String youtubeId = youtubeIdFromFilename(filename);
					int status = cursor.getInt(cursor.getColumnIndex(DownloadManager.COLUMN_STATUS));
					if (youtubeId != null) {
						int pct = -1;
						switch (status) {
						case DownloadManager.STATUS_FAILED:
						case DownloadManager.STATUS_PENDING:
						default:
							pct = 0;
							break;
						case DownloadManager.STATUS_PAUSED:
						case DownloadManager.STATUS_RUNNING:
							long bytes_downloaded = cursor.getInt(cursor.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR));
							long size = cursor.getInt(cursor.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES));
							pct = (int) (size==0? 0: 100 * bytes_downloaded / size);
							break;
						case DownloadManager.STATUS_SUCCESSFUL:
							pct = 100;
							break;
						}
						update.put(youtubeId, pct);
					}
				}
				cursor.close();
				
				return update;
			}
			
			return null;
		}
		
		@Override
		protected void onPostExecute(HashMap<String, Integer> update) {
			if (update != null) {
				doDownloadProgressUpdate(update);
			}
			// We've just finished polling. This will be set true again soon enough if a download
			// is in progress; we'll get a MODIFY event in the DownloadsObserver.
			currentPoller = null;
			shouldPoll = false;
		}
	}
	
	private Runnable pollerPoller = new Runnable() {
		@Override
		public void run() {
			if (shouldPoll && currentPoller == null) {
				currentPoller = new Poller();
				currentPoller.executeOnExecutor(pollerExecutor);
			}
			handler.postDelayed(this, 1000);
		}
	};
	
	/**
	 * Watches our downloads directory for changes.
	 * 
	 * This lets us know if a user has deleted a video or moved it away while the
	 * app is running. Update the db, set download_status to NOT_STARTED. For changes
	 * while we're NOT running, just check on startup.
	 * 
	 * This can also give us an indication that a current download has been paused
	 * via CLOSE_WRITE.
	 * 
	 * We know a download has begun the first time we see OPEN after it is enqueued.
	 */
	private final class DownloadsObserver extends FileObserver {
		
		 // From http://rswiki.csie.org/lxr/http/source/include/linux/inotify.h?a=m68k#L45
//			 29 #define IN_ACCESS               0x00000001      /* File was accessed */
//			 30 #define IN_MODIFY               0x00000002      /* File was modified */
//			 31 #define IN_ATTRIB               0x00000004      /* Metadata changed */
//			 32 #define IN_CLOSE_WRITE          0x00000008      /* Writtable file was closed */
//			 33 #define IN_CLOSE_NOWRITE        0x00000010      /* Unwrittable file closed */
//			 34 #define IN_OPEN                 0x00000020      /* File was opened */
//			 35 #define IN_MOVED_FROM           0x00000040      /* File was moved from X */
//			 36 #define IN_MOVED_TO             0x00000080      /* File was moved to Y */
//			 37 #define IN_CREATE               0x00000100      /* Subfile was created */
//			 38 #define IN_DELETE               0x00000200      /* Subfile was deleted */
//			 39 #define IN_DELETE_SELF          0x00000400      /* Self was deleted */
//			 40 #define IN_MOVE_SELF            0x00000800      /* Self was moved */
//			 41 
//			 42 /* the following are legal events.  they are sent as needed to any watch */
//			 43 #define IN_UNMOUNT              0x00002000      /* Backing fs was unmounted */
//			 44 #define IN_Q_OVERFLOW           0x00004000      /* Event queued overflowed */
//			 45 #define IN_IGNORED              0x00008000      /* File was ignored */
//			 46 
//			 47 /* helper events */
//			 48 #define IN_CLOSE                (IN_CLOSE_WRITE | IN_CLOSE_NOWRITE) /* close */
//			 49 #define IN_MOVE                 (IN_MOVED_FROM | IN_MOVED_TO) /* moves */
//			 50 
//			 51 /* special flags */
//			 52 #define IN_ONLYDIR              0x01000000      /* only watch the path if it is a directory */
//			 53 #define IN_DONT_FOLLOW          0x02000000      /* don't follow a sym link */
//			 54 #define IN_EXCL_UNLINK          0x04000000      /* exclude events on unlinked objects */
//			 55 #define IN_MASK_ADD             0x20000000      /* add to the mask of an already existing watch */
//			 56 #define IN_ISDIR                0x40000000      /* event occurred against dir */
//			 57 #define IN_ONESHOT              0x80000000      /* only send event once */
//		
//			 64 #define IN_ALL_EVENTS   (IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | \
//					 65                          IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | \
//					 66                          IN_MOVED_TO | IN_DELETE | IN_CREATE | IN_DELETE_SELF | \
//					 67                          IN_MOVE_SELF)
	
			
		private static final int flags =
				FileObserver.CLOSE_WRITE
				| FileObserver.OPEN
				| FileObserver.MODIFY
				| FileObserver.DELETE
				| FileObserver.MOVED_FROM;
		// Received three of these after the delete event while deleting a video through a separate file manager app:
		// 01-16 15:52:27.627: D/APP(4316): DownloadsObserver: onEvent(1073741856, null)
		// This is 0x40000020 : IN_ISDIR|IN_OPEN
		
		public DownloadsObserver(String path) {
			super(path, flags);
		}

		@Override
		public void onEvent(int event, final String path) {
			if (path == null) {
				return;
			}
			
			switch (event & FileObserver.ALL_EVENTS) {
			case FileObserver.CLOSE_WRITE:
				// Download complete, or paused when wifi is disconnected. Possibly reported more than once in a row.
				// Useful for noticing when a download has been paused. For completions, register a receiver for 
				// DownloadManager.ACTION_DOWNLOAD_COMPLETE.
				break;
			case FileObserver.OPEN:
				// Called for both read and write modes.
				// Useful for noticing a download has been started or resumed.
				break;
			case FileObserver.DELETE:
			case FileObserver.MOVED_FROM:
				// This video is lost never to return. Remove it.
				handler.post(new Runnable() {
					@Override
					public void run() {
						DatabaseHelper helper = dataService.getHelper();
						helper.removeDownloadFromDownloadManager(helper.getVideoForFilename(path));
					}
				});
				break;
			case FileObserver.MODIFY:
				// Called very frequently while a download is ongoing (~1 per ms).
				// This could be used to trigger a progress update, but that should probably be done less often than this.
				shouldPoll = true;
				break;
			}
		}
		
	}
	
	/**
	 * Creates a {@link DownloadProgressPoller} and a background thread for running it. Registers
	 * our {@link BroadcastReceiver} for {@link DownloadManager} updates.
	 * 
	 * @param dataService A {@link Context} with which to register our {@link BroadcastReceiver}.
	 */
	public OfflineVideoManager(KADataService dataService) {
		this.dataService = dataService;
		broadcastManager = LocalBroadcastManager.getInstance(dataService);
		
		try {
			videoDao = dataService.getHelper().getVideoDao();
		} catch (SQLException e) {
			throw new RuntimeException(e);
		}

		fileObserver = new DownloadsObserver(this.getDownloadDir().getAbsolutePath());
		fileObserver.startWatching();
		
		handler.postDelayed(pollerPoller, 1000);
		resume();
	}

	/* ***********************  PUBLIC   ****************************/

	/**
	 * Populate our lists of downloaded and downloading videos from
	 * {@link DownloadManager}.
	 */
	public void resume() {
		dataService.getHelper().syncWithDownloadManager();
	}
	
	/**
	 * Unregister callbacks.
	 */
	public void destroy() {
		fileObserver.stopWatching();
		handler.removeCallbacks(pollerPoller);
	}
	
	/**
	 * Cancel ongoing and enqueued video downloads.
	 */
	public void cancelAllVideoDownloads() {
		final DownloadManager dlm = getDownloadManager();
		final DownloadManager.Query q = new DownloadManager.Query();
		q.setFilterByStatus(DownloadManager.STATUS_FAILED | DownloadManager.STATUS_PAUSED
				| DownloadManager.STATUS_PENDING | DownloadManager.STATUS_RUNNING);
		
		// Cancel all tasks - we don't want any more downloads enqueued, and we are
		// beginning a cancel task so we don't need any previous one.
		queueExecutor.shutdownNow();
		queueExecutor = Executors.newSingleThreadExecutor();
		
		new AsyncTask<Void, Void, Integer>() {
			@Override
			protected void onPreExecute() {
				doToast("Stopping downloads...");
			}
			@Override
			protected Integer doInBackground(Void... arg) {
				int result = 0;
				if (isCancelled()) return result;
				
				Cursor c = dlm.query(q);
				Long[] removed = new Long[c.getCount()];
				int i = 0;
				while (c.moveToNext()) {
					if (isCancelled()) break;
					
					long id = c.getLong(c.getColumnIndex(DownloadManager.COLUMN_ID));
					removed[i++] = id;
					dlm.remove(id);
					result++;
				}
				c.close();
				
				UpdateBuilder<Video, String> u = videoDao.updateBuilder();
				try {
					u.where().in("dlm_id", (Object[]) removed);
					u.updateColumnValue("download_status", Video.DL_STATUS_NOT_STARTED);
					u.update();
				} catch (SQLException e) {
					e.printStackTrace();
				}
				
				return result;
			}
			@Override
			protected void onPostExecute(Integer result) {
				if (result > 0) {
					doToast(result + " downloads cancelled.");
				} else {
					doToast("No downloads in queue.");
				}
				doOfflineVideoSetChanged();
			}
			@Override
			protected void onCancelled(Integer result) {
				if (result > 0) {
					doToast(result + " downloads cancelled.");
				}
				doOfflineVideoSetChanged();
			}
		}.executeOnExecutor(queueExecutor);
	}
	
	/**
	 * Deletes the OfflineVideos from the filesystem. Cancels downloads if in progress.
	 * 
	 * @param toDelete the OfflineVideos to delete
	 */
	public void deleteOfflineVideos(Set<Video> toDelete) {
		if (toDelete == null || toDelete.size() == 0) return;
		
		// Remove from download manager if applicable, and update internal state.
		for (Video v : toDelete) {
			dataService.getHelper().removeDownloadFromDownloadManager(v);
		}
		
		// Update listeners and continue downloading.
		doToast(dataService.getString(R.string.msg_deleted));
		doOfflineVideoSetChanged();
	}
	
	/**
	 * Begin a video download by creating a {@link DownloadManager.Request} 
	 * and enqueuing it with the {@link DownloadManager}.
	 * 
	 * @param video The {@link Video} to download.
	 * @return False if external storage could not be loaded and thus the download cannot begin, true otherwise.
	 */
	public boolean downloadVideo(Video video) {
		// TODO 
//		dataService.getCaptionManager().getCaptionsForYoutubeId(video.getYoutube_id());
		
		Log.d(LOG_TAG, "downloadVideo");
		
		if (hasDownloadBegun(video)) return true;
		
		File file = getDownloadDir();
		if (file != null) {
			file = new File(file, getFilenameForOfflineVideo(video));
			file.delete(); //if it exists -- otherwise, this does nothing.
			Log.d(LOG_TAG, "starting download to file: " + file.getPath());
			DownloadManager mgr = getDownloadManager();
			String url = video.getMp4url();
			
			if (url == null) {
				//WHYYYY
				//TODO toast there is no download url for this video --- can we even view it? which video is this??
				// TODO : try m3u8 instead?
				Log.e(LOG_TAG, "NO DOWNLOAD URL FOR VIDEO " + video.getTitle());
				return false;
			}
			
			Uri requestUri = Uri.parse(url);
			DownloadManager.Request request = new DownloadManager.Request(requestUri);
			Uri localUri = Uri.fromFile(file);
	        request.setDestinationUri(localUri);
			request.setDescription(file.getName());
			request.setTitle(video.getTitle());
			request.setVisibleInDownloadsUi(false);
			long id = mgr.enqueue(request);
			Log.d(LOG_TAG, "download request ENQUEUED: " + id);
			
			// mark download as begun
			try {
				videoDao.refresh(video);
				video.setDownload_status(Video.DL_STATUS_IN_PROGRESS);
				video.setDlm_id(id);
				videoDao.update(video);
			} catch (SQLException e) {
				e.printStackTrace();
			}
			
			return true;
		} else {
			showDownloadError();
			return false;
		}
	}
	
	public void downloadAll(final Collection<Video> videos) {
		Log.d(LOG_TAG, "downloadAll (" + videos.size() + ")");
		final int size = videos.size();
		new AsyncTask<Void, Void, Integer>() {
			@Override
			protected void onPreExecute() {
				doToast(String.format("Downloading %d videos...", size));
			}
			@Override
			protected Integer doInBackground(Void... arg) {
				int result = 0;
				for (Video v : videos) {
					if (isCancelled()) break;
					
					result++;
					downloadVideo(v);
				}
				return result;
			}
		}.executeOnExecutor(queueExecutor);
	}
	
	/**
	 * Show a toast explaining that a {@link Video} could not be downloaded. 
	 */
	private void showDownloadError() {
		long time = SystemClock.uptimeMillis();
		if (time - lastErrorToastTime < MIN_ERROR_TOAST_INTERVAL) return;
		lastErrorToastTime = time;
		doToast("Error writing to storage. Please try again later.");
	}
	
	
	
	
	
	private void doToast(String message) {
		Intent intent = new Intent(ACTION_TOAST);
		intent.putExtra(EXTRA_MESSAGE, message);
		broadcastManager.sendBroadcast(intent);
	}
	
	private void doDownloadProgressUpdate(HashMap<String, Integer> youtubeIdToPct) {
		Log.d(LOG_TAG, "doDownloadProgressUpdate");
		Intent intent = new Intent(ACTION_DOWNLOAD_PROGRESS_UPDATE);
		intent.putExtra(EXTRA_STATUS, youtubeIdToPct);
		broadcastManager.sendBroadcast(intent);
	}
	
	/**
	 * Refresh our offline video lists, and call 
	 * {@link OfflineVideoSetChangedListener#onOfflineVideoSetChanged()} 
	 * on each registered {@link OfflineVideoSetChangedListener}.
	 */
	private void doOfflineVideoSetChanged() {
		Log.d(LOG_TAG, "Offline Video Set Changed");
		
		Intent intent = new Intent(Constants.ACTION_OFFLINE_VIDEO_SET_CHANGED);
		broadcastManager.sendBroadcast(intent);
	}
	
	
	
	
	
	
	
	/**
	 * Get whether a {@link Video} has begun downloading.
	 * 
	 * @param video The {@link Video} to check.
	 * @return True if the download has begun, false otherwise.  This means true if a download is complete, and false if the download is enqueued but not yet begun.
	 */
	public boolean hasDownloadBegun(Video video) {
		Log.d(LOG_TAG, "hasDownloadBegun");
		DownloadManager.Query q = new DownloadManager.Query();
		long dlm_id = video.getDlm_id();
		if (dlm_id <= 0) {
			return false;
		}
		q.setFilterById(dlm_id);
		q.setFilterByStatus(DownloadManager.STATUS_PAUSED
				| DownloadManager.STATUS_RUNNING | DownloadManager.STATUS_SUCCESSFUL);
		Cursor c = getDownloadManager().query(q);
		boolean result = c.getCount() > 0;
		c.close();
		return result;
	}
	
	/**
	 * Get a filename for the given {@link Video}.
	 * 
	 * @param v The {@link Video} for which to get a filename.
	 * @return The filename as a String.
	 */
	private String getFilenameForOfflineVideo(Video v) {
		return v.getYoutube_id() + ".mp4";
	}
	
	private String buildDownloadCountQuery(int depth) {
		Log.d(LOG_TAG, "buildQuery");
		String sql;
		if (depth == 0) {
			sql = "select count(video._id) from video,topicvideo where topicvideo.topic_id=? and topicvideo.video_id = video.readable_id";
		} else if (depth == 1) {
			sql = "select count(video._id) from video,topicvideo,topic as t1 where t1.parentTopic_id=? and topicvideo.topic_id=t1._id and topicvideo.video_id = video.readable_id";
		} else {
		
			// Depth 2~n
			sql = "select count(video._id) from video,topicvideo ,topic as t1";
			for (int i = 2; i < depth; ++i) {
				sql += String.format(",topic as t%d ", i);
			}
			sql += " where topicvideo.topic_id=t1._id and topicvideo.video_id=video.readable_id and t1.parentTopic_id=";
			
			for (int i = 2; i < depth; ++i) {
				sql += String.format("t%d._id and t%d.parentTopic_id=", i, i);
			}
			sql += "?";
			// Output of above with depth=6
	//		String out = "select count(video._id) from video,topicvideo ,topic as t1,topic as t2 ,topic as t3 ,topic as t4 ,topic as t5 " +
	//				"where topicvideo.topic_id=t1._id and topicvideo.video_id=video.readable_id  and t1.parentTopic_id=t2._id and t2.parentTopic_id=t3._id and t3.parentTopic_id=t4._id and t4.parentTopic_id=t5._id and t5.parentTopic_id=?";
			
			// Samples with "root"
			// depth = 1
	//		sql = "select count(video._id) from video,topicvideo,topic as t1 " +
	//				"where topicvideo.video_id = video.readable_id and topicvideo.topic_id=t1._id and t1.parentTopic_id='root'";
			
			// depth = 2
	//		sql = "select count(video._id) from video,topicvideo,topic as t1,topic as t2 " +
	//				"where topicvideo.video_id = video.readable_id and topicvideo.topic_id=t1._id and t1.parentTopic_id=t2._id and t2.parentTopic_id='root'  ";
		
		}
		
		sql += " and video.download_status=" + String.valueOf(Video.DL_STATUS_COMPLETE);
		
		Log.d(LOG_TAG, "  --> " + sql);
		return sql;
	}
	
	public int getDownloadCountForTopic(SQLiteOpenHelper dbh, String topicId, int depth) {
		Log.d(LOG_TAG, "getDownloadCountForTopic");
		
		int result = 0;
		SQLiteDatabase db = dbh.getReadableDatabase();
		
		for (int i = 0; i < depth; ++i) {
			String sql = buildDownloadCountQuery(i);
			Cursor c = db.rawQuery(sql, new String[] {topicId});
			c.moveToFirst();
			result += c.getInt(0);
			Log.d(LOG_TAG, " result is " + result);
			c.close();
		}
		
		return result;
	}
	
	
	
	
	public static String youtubeIdFromFilename(String filename) {
		if (filename == null) return null;
		
		String youtubeId = null;
		Matcher m = filenamePattern.matcher(filename);
		if (m.find()) {
			youtubeId = m.group(1);
		}
		return youtubeId;
	}
	
	
	
	private DownloadManager getDownloadManager() {
		return (DownloadManager) dataService.getSystemService(Context.DOWNLOAD_SERVICE);
	}
	
	private File getDownloadDir() {
		File file = dataService.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS);
		return file;
	}
	
} public class ThumbnailManager {
	
	public static final String LOG_TAG = ThumbnailManager.class.getSimpleName();
	private static final int CONNECT_TIMEOUT = 3000;

	private static ThumbnailManager sharedInstance;
	
	/* ***********************   PRIVATE   ***************************/
	
	private final KADataService dataService;
	private final ConnectivityManager connectivityManager;
	private final LruCache<Thumbnail, Bitmap> cache;
	private final DiskLruCache diskCache;
	
	private boolean isDestroyed;
	
	public static ThumbnailManager getSharedInstance(KADataService dataService) {
		if (sharedInstance == null || sharedInstance.isDestroyed) {
			sharedInstance = new ThumbnailManager(dataService);
		}
		return sharedInstance;
	}
	
	private ThumbnailManager(final KADataService dataService) {
		this.dataService = dataService;
		connectivityManager = (ConnectivityManager) dataService.getSystemService(Context.CONNECTIVITY_SERVICE);

		cache = prepareCache();
		diskCache = prepareDiskCache();
	}
	
	private DiskLruCache prepareDiskCache() {
	    int v = 0;
	    try {
	        v = dataService.getPackageManager().getPackageInfo(dataService.getPackageName(), 0).versionCode;
	    } catch (NameNotFoundException e) {
	        // Huh? Really?
	    }
	    
	    // TODO : allow user to configure this.
	    long maxSize = 1024 * 1024 * 1024;
	    
	    File cacheDir = new File(dataService.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS), "thumbnail_cache");
	    int valueCount = 8;
	    
		try {
			// TODO : This is slow on first run. Look into improving that.
			return DiskLruCache.open(cacheDir, v, valueCount, maxSize);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return null;
	}
	
	private LruCache<Thumbnail, Bitmap> prepareCache() {
		// Total available heap size. This changes based on manifest android:largeHeap="true". (Fire HD, transformer both go from 48MB to 256MB)
		Runtime rt = Runtime.getRuntime();
		long maxMemory = rt.maxMemory();
		Log.v(LOG_TAG, "maxMemory:" + Long.toString(maxMemory));
		
		// Want to use at most about 1/2 of available memory for thumbs.
		// In SAT Math category (116 videos), with a heap size of 48MB, this setting
		// allows 109 thumbs to be cached resulting in total heap usage around 34MB.
		long usableMemory = maxMemory / 2;
		
		return new LruCache<Thumbnail, Bitmap>((int) usableMemory) {
			@Override
			protected int sizeOf(Thumbnail key, Bitmap value) {
				return value.getByteCount();
			}
			
			@Override
			protected void entryRemoved(boolean evicted, Thumbnail key, Bitmap oldValue, Bitmap newValue) {
				if (oldValue != newValue) {
					oldValue.recycle();
				}
			}
		};
		
	}
	
	/* ***********************   PUBLIC   ***************************/

	public void destroy() {
		cache.evictAll();
		if (diskCache != null) {
			try {
				diskCache.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		isDestroyed = true;
	}
	
	/**
	 * Cache of thumbnails by youtube id.
	 * @return
	 */
	public LruCache<Thumbnail, Bitmap> getCache() {
		return cache;
	}
	
	/**
	 * Get the thumbnail for the video with the given youtube id, or start a download if it isn't yet stored locally.
	 * 
	 * @param y_id The youtube id of the video whose thumbnail we need.
	 * @return The thumbnail as a {@link Bitmap}, or {@code null} if none exists.
	 */
	
	public Bitmap getThumbnail(String y_id, byte quality, boolean useCache) {
		
		Bitmap result = null;
		
		if (useCache) {
			Thumbnail thumbnail = new Thumbnail(y_id, quality);
			result = cache.get(thumbnail);
		}
		
		if (result == null) {
			result = getThumbnail(y_id, quality);
		}
		return result;
	}
	
	private int indexForAvailability(byte q) {
		switch (q) {
		case Thumbnail.QUALITY_LOW:
			return 4;
		case Thumbnail.QUALITY_MEDIUM:
			return 5;
		case Thumbnail.QUALITY_HIGH:
			return 6;
		case Thumbnail.QUALITY_SD:
			return 7;
		default:
			throw new IllegalArgumentException("invalid thumb quality");
		}
	}
	private int indexForQuality(byte q) {
		switch (q) {
		case Thumbnail.QUALITY_LOW:
			return 0;
		case Thumbnail.QUALITY_MEDIUM:
			return 1;
		case Thumbnail.QUALITY_HIGH:
			return 2;
		case Thumbnail.QUALITY_SD:
			return 3;
		default:
			throw new IllegalArgumentException("invalid thumb quality");
		}
	}
	
	public Bitmap getThumbnailFromDiskCache(String youtubeId, byte quality) {
		String key = youtubeId.toLowerCase(Locale.US);
		Bitmap result = null;
		DiskLruCache.Snapshot snap = null;
		DiskLruCache.Editor editor = null;
		
		// Ensure we have a cache entry for this youtube id.
		try {
			// null while another editor is open
			while ((editor = diskCache.edit(key)) == null) { }
			
			if (editor.getString(indexForAvailability(Thumbnail.QUALITY_HIGH)) == null) {
				// values only null if they've never been set, so this must be a new entry
				editor.set(indexForQuality(Thumbnail.QUALITY_HIGH), "");
				editor.set(indexForAvailability(Thumbnail.QUALITY_HIGH), String.valueOf(Thumbnail.AVAILABILITY_UNKNOWN));
				editor.set(indexForQuality(Thumbnail.QUALITY_MEDIUM), "");
				editor.set(indexForAvailability(Thumbnail.QUALITY_MEDIUM), String.valueOf(Thumbnail.AVAILABILITY_UNKNOWN));
				editor.set(indexForQuality(Thumbnail.QUALITY_LOW), "");
				editor.set(indexForAvailability(Thumbnail.QUALITY_LOW), String.valueOf(Thumbnail.AVAILABILITY_UNKNOWN));
				editor.set(indexForQuality(Thumbnail.QUALITY_SD), "");
				editor.set(indexForAvailability(Thumbnail.QUALITY_SD), String.valueOf(Thumbnail.AVAILABILITY_UNKNOWN));
				editor.commit();
			}
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if (editor != null) editor.abortUnlessCommitted();
		}
		
		while (quality >= Thumbnail.QUALITY_LOW && result == null) {
			
			try {
				// Try getting a bitmap for this quality from disk.
				snap = diskCache.get(key);
				InputStream is = snap.getInputStream(indexForQuality(quality));
				try {
					result = BitmapFactory.decodeStream(is);
				} finally {
					if (is != null) {
						try { is.close(); } catch (IOException ex) { }
					}
				}
				if (result != null) {
					return result;
				}
			
				// If none exists, try fetching it if we haven't before.
				int availability = Integer.parseInt(snap.getString(indexForAvailability(quality)));
				if (availability != Thumbnail.AVAILABILITY_UNAVAILABLE) {
					NetworkInfo activeNetwork = connectivityManager.getActiveNetworkInfo();
					if (activeNetwork != null && activeNetwork.isConnected()) {
						try {
							result = bitmap_from_url(Thumbnail.getDownloadUrl(dataService, quality, youtubeId));
						} catch (MalformedURLException e) {
							e.printStackTrace();
						} catch (IOException e) {
							// FileNotFoundException on 404. Mark as unavailable.
							if (e instanceof FileNotFoundException) {
								try {
									while ((editor = diskCache.edit(key)) == null) { }
									editor.set(indexForAvailability(quality), String.valueOf(Thumbnail.AVAILABILITY_UNAVAILABLE));
									editor.commit();
								} catch (IOException ex) {
									ex.printStackTrace();
								} finally {
									if (editor != null) editor.abortUnlessCommitted();
								}
							} else {
								e.printStackTrace();
							}
						}
					}
					
					// If we receive a thumbnail response, store it in the cache and return it.
					if (result != null) {
						try {
							while ((editor = diskCache.edit(key)) == null) { }
							OutputStream os = editor.newOutputStream(indexForQuality(quality));
							try {
								result.compress(Bitmap.CompressFormat.PNG, 100, os);
							} finally {
								if (os != null) os.close();
							}
							editor.set(indexForAvailability(quality), String.valueOf(Thumbnail.AVAILABILITY_AVAILABLE));
							editor.commit();
						} catch (IOException e) {
							e.printStackTrace();
						} finally {
							if (editor != null) editor.abortUnlessCommitted();
						}
						return result;
					}
				}
				
			} catch (IOException e) {
				e.printStackTrace();
			} finally {
				if (snap != null) snap.close();
			}
			
			quality--;
		}
		
		return result;
	}
	
	public Bitmap getThumbnail(String y_id, byte quality) {
		Log.v(LOG_TAG, ".getThumbnailForYoutubeId");
		return getThumbnailFromDiskCache(y_id, quality);
	}

	/**
	 * Attempt to download a bitmap from the given url.
	 * 
	 * @param url The url of the thumbnail to download.
	 * @return A {@link Bitmap} of the thumbnail, or {@code null} if it cannot be downloaded and is not cached locally.
	 * @throws java.net.MalformedURLException if the url is malformed!
	 * @throws java.io.IOException if a connection cannot be opened to the given url, or if an IOException is thrown by {@link ResponseCache} or by {@link CacheResponse}.
	 */
	public static Bitmap bitmap_from_url(String url) throws java.net.MalformedURLException, IOException {
	    HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
	    connection.setConnectTimeout(CONNECT_TIMEOUT);
	    connection.setUseCaches(true);
	    
		InputStream input = null;
	    try {
	    	connection.connect();
		    input = connection.getInputStream();
		    return BitmapFactory.decodeStream(input);
	    } catch (SocketTimeoutException e) {
	    	e.printStackTrace();
	    	return null;
	    } finally {
	    	if (input != null) input.close();
	    }
	}
	
	
	/*
	 "media$thumbnail": [
		{
			"url": "http://i.ytimg.com/vi/l-QFT7XNeb4/default.jpg",
			"height": 90,
			"width": 120,
			"time": "00:08:31",
			"yt$name": "default"
		},
		{
			"url": "http://i.ytimg.com/vi/l-QFT7XNeb4/mqdefault.jpg",
			"height": 180,
			"width": 320,
			"yt$name": "mqdefault"
		},
		{
			"url": "http://i.ytimg.com/vi/l-QFT7XNeb4/hqdefault.jpg",
			"height": 360,
			"width": 480,
			"yt$name": "hqdefault"
		},
		{
			"url": "http://i.ytimg.com/vi/l-QFT7XNeb4/sddefault.jpg",
			"height": 480,
			"width": 640,
			"yt$name": "sddefault"
		},
		{
			"url": "http://i.ytimg.com/vi/l-QFT7XNeb4/1.jpg",
			"height": 90,
			"width": 120,
			"time": "00:04:15.500",
			"yt$name": "start"
		},
		{
			"url": "http://i.ytimg.com/vi/l-QFT7XNeb4/2.jpg",
			"height": 90,
			"width": 120,
			"time": "00:08:31",
			"yt$name": "middle"
		},
		{
			"url": "http://i.ytimg.com/vi/l-QFT7XNeb4/3.jpg",
			"height": 90,
			"width": 120,
			"time": "00:12:46.500",
			"yt$name": "end"
		}
	],
	*/
	

}
import static com.concentricsky.android.khanacademy.Constants.ACTION_LIBRARY_UPDATE;
import static com.concentricsky.android.khanacademy.Constants.ACTION_UPDATE_DOWNLOAD_STATUS;
import static com.concentricsky.android.khanacademy.Constants.EXTRA_FORCE;
import static com.concentricsky.android.khanacademy.Constants.EXTRA_ID;
import static com.concentricsky.android.khanacademy.Constants.RESULT_CODE_FAILURE;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import org.json.JSONException;
import org.json.JSONObject;
import android.app.DownloadManager;
import android.app.Notification;
import android.app.NotificationManager;
import android.app.PendingIntent;
import android.app.PendingIntent.CanceledException;
import android.app.Service;
import android.content.Context;
import android.content.Intent;
import android.database.Cursor;
import android.os.AsyncTask;
import android.os.Binder;
import android.os.IBinder;
import android.support.v4.app.NotificationCompat;
import android.support.v4.content.LocalBroadcastManager;
import com.concentricsky.android.khan.R;
import com.concentricsky.android.khanacademy.Constants;
import com.concentricsky.android.khanacademy.data.db.DatabaseHelper;
import com.concentricsky.android.khanacademy.data.db.Topic;
import com.concentricsky.android.khanacademy.data.db.Video;
import com.concentricsky.android.khanacademy.data.remote.KAAPIAdapter;
import com.concentricsky.android.khanacademy.data.remote.LibraryUpdaterTask;
import com.concentricsky.android.khanacademy.util.CaptionManager;
import com.concentricsky.android.khanacademy.util.Log;
import com.concentricsky.android.khanacademy.util.ObjectCallback;
import com.concentricsky.android.khanacademy.util.OfflineVideoManager;
import com.concentricsky.android.khanacademy.util.ThumbnailManager;
import com.j256.ormlite.android.apptools.OpenHelperManager;
import com.j256.ormlite.dao.Dao;
import com.j256.ormlite.stmt.PreparedQuery;
import com.j256.ormlite.stmt.UpdateBuilder;
/*
 	Viewer for Khan Academy
    Copyright (C) 2012 Concentric Sky, Inc.

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package com.concentricsky.android.khanacademy.data;






/**
 * Manages all data for the application.
 * 
 * This service holds a reference to a database Helper, the Khan Academy API adapter,
 * and a local storage adapter for downloaded videos. It also sets a default HTTP
 * response cache.
 * 
 * We implement both onStartCommand and onBind.  Activities should bind to this service
 * with bindService to use it as a persistent data backend.  Meanwhile, we can call
 * startService to start long-running operations like video downloads and library
 * updates.
 * 
 * @author austinlally
 *
 */
public class KADataService extends Service {
    
	public final String LOG_TAG = getClass().getSimpleName();
	
	public static final int RESULT_SUCCESS = 0;
	public static final int RESULT_ERROR = 1;
	public static final int RESULT_CANCELLED = 2;
	
	private KADataBinder mBinder = new KADataBinder(this);
    private DatabaseHelper helper;
    private KAAPIAdapter api;