Changed cached OsuFile loading.

Instead of being loaded individually from the database, OsuFiles are now loaded in batch by traversing the database.  In most cases, this should cause a ~10% loading speed improvement.  Defaults to the previous approach if few OsuFiles are being loaded.

Other changes:
- Don't drop indexes upon batch insertion if few entries are being inserted.
- Trying to increase fetch size with setFetchSize(100).
- Added more parser statuses.

Signed-off-by: Jeffrey Han <itdelatrisu@gmail.com>

src/itdelatrisu/opsu/OsuParser.java

@@ -31,6 +31,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 
 import org.newdawn.slick.Color;
@@ -55,8 +56,11 @@ public class OsuParser {
 	/** The total number of directories to parse. */
 	private static int totalDirectories = -1;
 
-	/** Whether or not the database is currently being updated. */
+	/** Parser statuses. */
-	private static boolean updatingDatabase = false;
+	public enum Status { NONE, PARSING, CACHE, INSERTING };
+
+	/** The current status. */
+	private static Status status = Status.NONE;
 
 	// This class should not be instantiated.
 	private OsuParser() {}
@@ -85,14 +89,19 @@ public class OsuParser {
 			return null;
 
 		// progress tracking
+		status = Status.PARSING;
 		currentDirectoryIndex = 0;
 		totalDirectories = dirs.length;
 
 		// get last modified map from database
 		Map<String, Long> map = OsuDB.getLastModifiedMap();
 
+		// OsuFile lists
+		List<ArrayList<OsuFile>> allOsuFiles = new LinkedList<ArrayList<OsuFile>>();
+		List<OsuFile> cachedOsuFiles = new LinkedList<OsuFile>();  // loaded from database
+		List<OsuFile> parsedOsuFiles = new LinkedList<OsuFile>();  // loaded from parser
+
 		// parse directories
-		LinkedList<OsuFile> parsedOsuFiles = new LinkedList<OsuFile>();
 		OsuGroupNode lastNode = null;
 		for (File dir : dirs) {
 			currentDirectoryIndex++;
@@ -120,7 +129,10 @@ public class OsuParser {
 					// check last modified times
 					long lastModified = map.get(path);
 					if (lastModified == file.lastModified()) {
-						osuFiles.add(OsuDB.getOsuFile(dir, file));
+						// add to cached beatmap list
+						OsuFile osu = new OsuFile(file);
+						osuFiles.add(osu);
+						cachedOsuFiles.add(osu);
 						continue;
 					} else
 						OsuDB.delete(dir.getName(), file.getName());
@@ -130,13 +142,17 @@ public class OsuParser {
 				// Change boolean to 'true' to parse them immediately.
 				OsuFile osu = parseFile(file, dir, osuFiles, false);
 
-				if (osu != null)
+				// add to parsed beatmap list
+				if (osu != null) {
+					osuFiles.add(osu);
 					parsedOsuFiles.add(osu);
+				}
 			}
-			if (!osuFiles.isEmpty()) {  // add entry if non-empty
+
+			// add group entry if non-empty
+			if (!osuFiles.isEmpty()) {
 				osuFiles.trimToSize();
-				Collections.sort(osuFiles);
+				allOsuFiles.add(osuFiles);
-				lastNode = OsuGroupList.get().addSongGroup(osuFiles);
 			}
 
 			// stop parsing files (interrupted)
@@ -144,16 +160,28 @@ public class OsuParser {
 				break;
 		}
 
+		// load cached entries from database
+		if (!cachedOsuFiles.isEmpty()) {
+			status = Status.CACHE;
+			OsuDB.load(cachedOsuFiles);
+		}
+
+		// add group entries to OsuGroupList
+		for (ArrayList<OsuFile> osuFiles : allOsuFiles) {
+			Collections.sort(osuFiles);
+			lastNode = OsuGroupList.get().addSongGroup(osuFiles);
+		}
+
 		// clear string DB
 		stringdb = new HashMap<String, String>();
 
-		// add entries to database
+		// add beatmap entries to database
 		if (!parsedOsuFiles.isEmpty()) {
-			updatingDatabase = true;
+			status = Status.INSERTING;
 			OsuDB.insert(parsedOsuFiles);
-			updatingDatabase = false;
 		}
 
+		status = Status.NONE;
 		currentFile = null;
 		currentDirectoryIndex = -1;
 		totalDirectories = -1;
@@ -197,7 +225,8 @@ public class OsuParser {
 								if (!osuFiles.isEmpty()) {
 									// if possible, reuse the same File object from another OsuFile in the group
 									File groupAudioFileName = osuFiles.get(0).audioFilename;
-									if (tokens[1].equalsIgnoreCase(groupAudioFileName.getName()))
+									if (groupAudioFileName != null &&
+									    tokens[1].equalsIgnoreCase(groupAudioFileName.getName()))
 										audioFileName = groupAudioFileName;
 								}
 								if (!audioFileName.isFile()) {
@@ -550,10 +579,6 @@ public class OsuParser {
 		if (parseObjects)
 			parseHitObjects(osu);
 
-		// add OsuFile to song group
-		if (osuFiles != null)
-			osuFiles.add(osu);
-
 		return osu;
 	}
 
@@ -666,10 +691,10 @@ public class OsuParser {
 	 * Returns the name of the current file being parsed, or null if none.
 	 */
 	public static String getCurrentFileName() {
-		if (updatingDatabase)
+		if (status == Status.PARSING)
-			return "";
-		else
 			return (currentFile != null) ? currentFile.getName() : null;
+		else
+			return (status == Status.NONE) ? null : "";
 	}
 
 	/**
@@ -684,9 +709,9 @@ public class OsuParser {
 	}
 
 	/**
-	 * Returns whether or not the beatmap database is currently being updated.
+	 * Returns the current parser status.
 	 */
-	public static boolean isUpdatingDatabase() { return updatingDatabase; }
+	public static Status getStatus() { return status; }
 
 	/**
 	 * Returns the String object in the database for the given String.

src/itdelatrisu/opsu/UI.java

@@ -409,7 +409,8 @@ public class UI {
 			text = "Unpacking new beatmaps...";
 			progress = OszUnpacker.getUnpackerProgress();
 		} else if ((file = OsuParser.getCurrentFileName()) != null) {
-			text = (OsuParser.isUpdatingDatabase()) ? "Updating database..." : "Loading beatmaps...";
+			text = (OsuParser.getStatus() == OsuParser.Status.INSERTING) ?
+					"Updating database..." : "Loading beatmaps...";
 			progress = OsuParser.getParserProgress();
 		} else if ((file = SoundController.getCurrentFileName()) != null) {
 			text = "Loading sounds...";

src/itdelatrisu/opsu/db/OsuDB.java

@@ -43,11 +43,17 @@ public class OsuDB {
 	 */
 	private static final String DATABASE_VERSION = "2014-03-04";
 
+	/** Minimum batch size to invoke batch loading. */
+	private static final int LOAD_BATCH_MIN = 100;
+
+	/** Minimum batch size to invoke batch insertion. */
+	private static final int INSERT_BATCH_MIN = 100;
+
 	/** Database connection. */
 	private static Connection connection;
 
 	/** Query statements. */
-	private static PreparedStatement insertStmt, selectStmt, lastModStmt, deleteMapStmt, deleteGroupStmt;
+	private static PreparedStatement insertStmt, selectStmt, selectAllStmt, lastModStmt, deleteMapStmt, deleteGroupStmt;
 
 	// This class should not be instantiated.
 	private OsuDB() {}
@@ -75,6 +81,7 @@ public class OsuDB {
 				"?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
 			);
 			lastModStmt = connection.prepareStatement("SELECT dir, file, lastModified FROM beatmaps");
+			selectAllStmt = connection.prepareStatement("SELECT * FROM beatmaps");
 			selectStmt = connection.prepareStatement("SELECT * FROM beatmaps WHERE dir = ? AND file = ?");
 			deleteMapStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ? AND file = ?");
 			deleteGroupStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ?");
@@ -191,8 +198,11 @@ public class OsuDB {
 			connection.setAutoCommit(false);
 
 			// drop indexes
-			String sql = "DROP INDEX IF EXISTS idx";
+			boolean recreateIndexes = (batch.size() >= INSERT_BATCH_MIN);
-			stmt.executeUpdate(sql);
+			if (recreateIndexes) {
+				String sql = "DROP INDEX IF EXISTS idx";
+				stmt.executeUpdate(sql);
+			}
 
 			// batch insert
 			for (OsuFile osu : batch) {
@@ -202,8 +212,10 @@ public class OsuDB {
 			insertStmt.executeBatch();
 
 			// re-create indexes
-			sql = "CREATE INDEX idx ON beatmaps (dir, file)";
+			if (recreateIndexes) {
-			stmt.executeUpdate(sql);
+				String sql = "CREATE INDEX idx ON beatmaps (dir, file)";
+				stmt.executeUpdate(sql);
+			}
 
 			// restore previous auto-commit mode
 			connection.commit();
@@ -263,66 +275,123 @@ public class OsuDB {
 	}
 
 	/**
-	 * Returns an OsuFile from the database, or null if any error occurred.
+	 * Loads OsuFile fields from the database.
-	 * @param dir the directory
+	 * @param osu the OsuFile object
-	 * @param file the file
-	 * @return the OsuFile with all fields filled, or null if any error occurred
 	 */
-	public static OsuFile getOsuFile(File dir, File file) {
+	public static void load(OsuFile osu) {
 		if (connection == null)
-			return null;
+			return;
 
 		try {
-			OsuFile osu = new OsuFile(file);
+			selectStmt.setString(1, osu.getFile().getParentFile().getName());
-			selectStmt.setString(1, dir.getName());
+			selectStmt.setString(2, osu.getFile().getName());
-			selectStmt.setString(2, file.getName());
 			ResultSet rs = selectStmt.executeQuery();
+			if (rs.next())
+				setOsuFileFields(rs, osu);
+			rs.close();
+		} catch (SQLException e) {
+			ErrorHandler.error("Failed to load OsuFile from database.", e, true);
+		}
+	}
+
+	/**
+	 * Loads OsuFile fields from the database in a batch.
+	 * @param batch a list of OsuFile objects
+	 */
+	public static void load(List<OsuFile> batch) {
+		if (connection == null)
+			return;
+
+		// batch size too small
+		int size = batch.size();
+		if (size < LOAD_BATCH_MIN) {
+			for (OsuFile osu : batch)
+				load(osu);
+			return;
+		}
+
+		try {
+			// create map
+			HashMap<String, HashMap<String, OsuFile>> map = new HashMap<String, HashMap<String, OsuFile>>();
+			for (OsuFile osu : batch) {
+				String parent = osu.getFile().getParentFile().getName();
+				String name = osu.getFile().getName();
+				HashMap<String, OsuFile> m = map.get(parent);
+				if (m == null) {
+					m = new HashMap<String, OsuFile>();
+					map.put(parent, m);
+				}
+				m.put(name, osu);
+			}
+
+			// iterate through database to load OsuFiles
+			int count = 0;
+			selectAllStmt.setFetchSize(100);
+			ResultSet rs = selectAllStmt.executeQuery();
 			while (rs.next()) {
-				osu.beatmapID = rs.getInt(4);
+				String parent = rs.getString(1);
-				osu.beatmapSetID = rs.getInt(5);
+				HashMap<String, OsuFile> m = map.get(parent);
-				osu.title = OsuParser.getDBString(rs.getString(6));
+				if (m != null) {
-				osu.titleUnicode = OsuParser.getDBString(rs.getString(7));
+					String name = rs.getString(2);
-				osu.artist = OsuParser.getDBString(rs.getString(8));
+					OsuFile osu = m.get(name);
-				osu.artistUnicode = OsuParser.getDBString(rs.getString(9));
+					if (osu != null) {
-				osu.creator = OsuParser.getDBString(rs.getString(10));
+						setOsuFileFields(rs, osu);
-				osu.version = OsuParser.getDBString(rs.getString(11));
+						if (++count >= size)
-				osu.source = OsuParser.getDBString(rs.getString(12));
+							break;
-				osu.tags = OsuParser.getDBString(rs.getString(13));
+					}
-				osu.hitObjectCircle = rs.getInt(14);
+				}
-				osu.hitObjectSlider = rs.getInt(15);
-				osu.hitObjectSpinner = rs.getInt(16);
-				osu.HPDrainRate = rs.getFloat(17);
-				osu.circleSize = rs.getFloat(18);
-				osu.overallDifficulty = rs.getFloat(19);
-				osu.approachRate = rs.getFloat(20);
-				osu.sliderMultiplier = rs.getFloat(21);
-				osu.sliderTickRate = rs.getFloat(22);
-				osu.bpmMin = rs.getInt(23);
-				osu.bpmMax = rs.getInt(24);
-				osu.endTime = rs.getInt(25);
-				osu.audioFilename = new File(dir, OsuParser.getDBString(rs.getString(26)));
-				osu.audioLeadIn = rs.getInt(27);
-				osu.previewTime = rs.getInt(28);
-				osu.countdown = rs.getByte(29);
-				osu.sampleSet = OsuParser.getDBString(rs.getString(30));
-				osu.stackLeniency = rs.getFloat(31);
-				osu.mode = rs.getByte(32);
-				osu.letterboxInBreaks = rs.getBoolean(33);
-				osu.widescreenStoryboard = rs.getBoolean(34);
-				osu.epilepsyWarning = rs.getBoolean(35);
-				osu.bg = OsuParser.getDBString(rs.getString(36));
-				osu.timingPointsFromString(rs.getString(37));
-				osu.breaksFromString(rs.getString(38));
-				osu.comboFromString(rs.getString(39));
 			}
 			rs.close();
-			return osu;
 		} catch (SQLException e) {
-			ErrorHandler.error("Failed to get OsuFile from database.", e, true);
+			ErrorHandler.error("Failed to load OsuFiles from database.", e, true);
-			return null;
 		}
 	}
 
+	/**
+	 * Sets all OsuFile fields using a given result set.
+	 * @param rs the result set containing the fields
+	 * @param osu the OsuFile
+	 * @throws SQLException 
+	 */
+	private static void setOsuFileFields(ResultSet rs, OsuFile osu) throws SQLException {
+		osu.beatmapID = rs.getInt(4);
+		osu.beatmapSetID = rs.getInt(5);
+		osu.title = OsuParser.getDBString(rs.getString(6));
+		osu.titleUnicode = OsuParser.getDBString(rs.getString(7));
+		osu.artist = OsuParser.getDBString(rs.getString(8));
+		osu.artistUnicode = OsuParser.getDBString(rs.getString(9));
+		osu.creator = OsuParser.getDBString(rs.getString(10));
+		osu.version = OsuParser.getDBString(rs.getString(11));
+		osu.source = OsuParser.getDBString(rs.getString(12));
+		osu.tags = OsuParser.getDBString(rs.getString(13));
+		osu.hitObjectCircle = rs.getInt(14);
+		osu.hitObjectSlider = rs.getInt(15);
+		osu.hitObjectSpinner = rs.getInt(16);
+		osu.HPDrainRate = rs.getFloat(17);
+		osu.circleSize = rs.getFloat(18);
+		osu.overallDifficulty = rs.getFloat(19);
+		osu.approachRate = rs.getFloat(20);
+		osu.sliderMultiplier = rs.getFloat(21);
+		osu.sliderTickRate = rs.getFloat(22);
+		osu.bpmMin = rs.getInt(23);
+		osu.bpmMax = rs.getInt(24);
+		osu.endTime = rs.getInt(25);
+		osu.audioFilename = new File(osu.getFile().getParentFile(), OsuParser.getDBString(rs.getString(26)));
+		osu.audioLeadIn = rs.getInt(27);
+		osu.previewTime = rs.getInt(28);
+		osu.countdown = rs.getByte(29);
+		osu.sampleSet = OsuParser.getDBString(rs.getString(30));
+		osu.stackLeniency = rs.getFloat(31);
+		osu.mode = rs.getByte(32);
+		osu.letterboxInBreaks = rs.getBoolean(33);
+		osu.widescreenStoryboard = rs.getBoolean(34);
+		osu.epilepsyWarning = rs.getBoolean(35);
+		osu.bg = OsuParser.getDBString(rs.getString(36));
+		osu.timingPointsFromString(rs.getString(37));
+		osu.breaksFromString(rs.getString(38));
+		osu.comboFromString(rs.getString(39));
+	}
+
 	/**
 	 * Returns a map of file paths ({dir}/{file}) to last modified times, or
 	 * null if any error occurred.
@@ -334,6 +403,7 @@ public class OsuDB {
 		try {
 			Map<String, Long> map = new HashMap<String, Long>();
 			ResultSet rs = lastModStmt.executeQuery();
+			lastModStmt.setFetchSize(100);
 			while (rs.next()) {
 				String path = String.format("%s/%s", rs.getString(1), rs.getString(2));
 				long lastModified = rs.getLong(3);
@@ -392,6 +462,7 @@ public class OsuDB {
 			insertStmt.close();
 			lastModStmt.close();
 			selectStmt.close();
+			selectAllStmt.close();
 			deleteMapStmt.close();
 			deleteGroupStmt.close();
 			connection.close();