diff --git a/src/itdelatrisu/opsu/OsuParser.java b/src/itdelatrisu/opsu/OsuParser.java index 67d2f515..f82c58bf 100644 --- a/src/itdelatrisu/opsu/OsuParser.java +++ b/src/itdelatrisu/opsu/OsuParser.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; +import java.util.List; import java.util.Map; import org.newdawn.slick.Color; @@ -55,8 +56,11 @@ public class OsuParser { /** The total number of directories to parse. */ private static int totalDirectories = -1; - /** Whether or not the database is currently being updated. */ - private static boolean updatingDatabase = false; + /** Parser statuses. */ + public enum Status { NONE, PARSING, CACHE, INSERTING }; + + /** The current status. */ + private static Status status = Status.NONE; // This class should not be instantiated. private OsuParser() {} @@ -85,14 +89,19 @@ public class OsuParser { return null; // progress tracking + status = Status.PARSING; currentDirectoryIndex = 0; totalDirectories = dirs.length; // get last modified map from database Map map = OsuDB.getLastModifiedMap(); + // OsuFile lists + List> allOsuFiles = new LinkedList>(); + List cachedOsuFiles = new LinkedList(); // loaded from database + List parsedOsuFiles = new LinkedList(); // loaded from parser + // parse directories - LinkedList parsedOsuFiles = new LinkedList(); OsuGroupNode lastNode = null; for (File dir : dirs) { currentDirectoryIndex++; @@ -120,7 +129,10 @@ public class OsuParser { // check last modified times long lastModified = map.get(path); if (lastModified == file.lastModified()) { - osuFiles.add(OsuDB.getOsuFile(dir, file)); + // add to cached beatmap list + OsuFile osu = new OsuFile(file); + osuFiles.add(osu); + cachedOsuFiles.add(osu); continue; } else OsuDB.delete(dir.getName(), file.getName()); @@ -130,13 +142,17 @@ public class OsuParser { // Change boolean to 'true' to parse them immediately. OsuFile osu = parseFile(file, dir, osuFiles, false); - if (osu != null) + // add to parsed beatmap list + if (osu != null) { + osuFiles.add(osu); parsedOsuFiles.add(osu); + } } - if (!osuFiles.isEmpty()) { // add entry if non-empty + + // add group entry if non-empty + if (!osuFiles.isEmpty()) { osuFiles.trimToSize(); - Collections.sort(osuFiles); - lastNode = OsuGroupList.get().addSongGroup(osuFiles); + allOsuFiles.add(osuFiles); } // stop parsing files (interrupted) @@ -144,16 +160,28 @@ public class OsuParser { break; } + // load cached entries from database + if (!cachedOsuFiles.isEmpty()) { + status = Status.CACHE; + OsuDB.load(cachedOsuFiles); + } + + // add group entries to OsuGroupList + for (ArrayList osuFiles : allOsuFiles) { + Collections.sort(osuFiles); + lastNode = OsuGroupList.get().addSongGroup(osuFiles); + } + // clear string DB stringdb = new HashMap(); - // add entries to database + // add beatmap entries to database if (!parsedOsuFiles.isEmpty()) { - updatingDatabase = true; + status = Status.INSERTING; OsuDB.insert(parsedOsuFiles); - updatingDatabase = false; } + status = Status.NONE; currentFile = null; currentDirectoryIndex = -1; totalDirectories = -1; @@ -197,7 +225,8 @@ public class OsuParser { if (!osuFiles.isEmpty()) { // if possible, reuse the same File object from another OsuFile in the group File groupAudioFileName = osuFiles.get(0).audioFilename; - if (tokens[1].equalsIgnoreCase(groupAudioFileName.getName())) + if (groupAudioFileName != null && + tokens[1].equalsIgnoreCase(groupAudioFileName.getName())) audioFileName = groupAudioFileName; } if (!audioFileName.isFile()) { @@ -550,10 +579,6 @@ public class OsuParser { if (parseObjects) parseHitObjects(osu); - // add OsuFile to song group - if (osuFiles != null) - osuFiles.add(osu); - return osu; } @@ -666,10 +691,10 @@ public class OsuParser { * Returns the name of the current file being parsed, or null if none. */ public static String getCurrentFileName() { - if (updatingDatabase) - return ""; - else + if (status == Status.PARSING) return (currentFile != null) ? currentFile.getName() : null; + else + return (status == Status.NONE) ? null : ""; } /** @@ -684,9 +709,9 @@ public class OsuParser { } /** - * Returns whether or not the beatmap database is currently being updated. + * Returns the current parser status. */ - public static boolean isUpdatingDatabase() { return updatingDatabase; } + public static Status getStatus() { return status; } /** * Returns the String object in the database for the given String. diff --git a/src/itdelatrisu/opsu/UI.java b/src/itdelatrisu/opsu/UI.java index 3c03f610..4ef0ec3d 100644 --- a/src/itdelatrisu/opsu/UI.java +++ b/src/itdelatrisu/opsu/UI.java @@ -409,7 +409,8 @@ public class UI { text = "Unpacking new beatmaps..."; progress = OszUnpacker.getUnpackerProgress(); } else if ((file = OsuParser.getCurrentFileName()) != null) { - text = (OsuParser.isUpdatingDatabase()) ? "Updating database..." : "Loading beatmaps..."; + text = (OsuParser.getStatus() == OsuParser.Status.INSERTING) ? + "Updating database..." : "Loading beatmaps..."; progress = OsuParser.getParserProgress(); } else if ((file = SoundController.getCurrentFileName()) != null) { text = "Loading sounds..."; diff --git a/src/itdelatrisu/opsu/db/OsuDB.java b/src/itdelatrisu/opsu/db/OsuDB.java index 1e5e085e..a46931fa 100644 --- a/src/itdelatrisu/opsu/db/OsuDB.java +++ b/src/itdelatrisu/opsu/db/OsuDB.java @@ -43,11 +43,17 @@ public class OsuDB { */ private static final String DATABASE_VERSION = "2014-03-04"; + /** Minimum batch size to invoke batch loading. */ + private static final int LOAD_BATCH_MIN = 100; + + /** Minimum batch size to invoke batch insertion. */ + private static final int INSERT_BATCH_MIN = 100; + /** Database connection. */ private static Connection connection; /** Query statements. */ - private static PreparedStatement insertStmt, selectStmt, lastModStmt, deleteMapStmt, deleteGroupStmt; + private static PreparedStatement insertStmt, selectStmt, selectAllStmt, lastModStmt, deleteMapStmt, deleteGroupStmt; // This class should not be instantiated. private OsuDB() {} @@ -75,6 +81,7 @@ public class OsuDB { "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" ); lastModStmt = connection.prepareStatement("SELECT dir, file, lastModified FROM beatmaps"); + selectAllStmt = connection.prepareStatement("SELECT * FROM beatmaps"); selectStmt = connection.prepareStatement("SELECT * FROM beatmaps WHERE dir = ? AND file = ?"); deleteMapStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ? AND file = ?"); deleteGroupStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ?"); @@ -191,8 +198,11 @@ public class OsuDB { connection.setAutoCommit(false); // drop indexes - String sql = "DROP INDEX IF EXISTS idx"; - stmt.executeUpdate(sql); + boolean recreateIndexes = (batch.size() >= INSERT_BATCH_MIN); + if (recreateIndexes) { + String sql = "DROP INDEX IF EXISTS idx"; + stmt.executeUpdate(sql); + } // batch insert for (OsuFile osu : batch) { @@ -202,8 +212,10 @@ public class OsuDB { insertStmt.executeBatch(); // re-create indexes - sql = "CREATE INDEX idx ON beatmaps (dir, file)"; - stmt.executeUpdate(sql); + if (recreateIndexes) { + String sql = "CREATE INDEX idx ON beatmaps (dir, file)"; + stmt.executeUpdate(sql); + } // restore previous auto-commit mode connection.commit(); @@ -263,66 +275,123 @@ public class OsuDB { } /** - * Returns an OsuFile from the database, or null if any error occurred. - * @param dir the directory - * @param file the file - * @return the OsuFile with all fields filled, or null if any error occurred + * Loads OsuFile fields from the database. + * @param osu the OsuFile object */ - public static OsuFile getOsuFile(File dir, File file) { + public static void load(OsuFile osu) { if (connection == null) - return null; + return; try { - OsuFile osu = new OsuFile(file); - selectStmt.setString(1, dir.getName()); - selectStmt.setString(2, file.getName()); + selectStmt.setString(1, osu.getFile().getParentFile().getName()); + selectStmt.setString(2, osu.getFile().getName()); ResultSet rs = selectStmt.executeQuery(); + if (rs.next()) + setOsuFileFields(rs, osu); + rs.close(); + } catch (SQLException e) { + ErrorHandler.error("Failed to load OsuFile from database.", e, true); + } + } + + /** + * Loads OsuFile fields from the database in a batch. + * @param batch a list of OsuFile objects + */ + public static void load(List batch) { + if (connection == null) + return; + + // batch size too small + int size = batch.size(); + if (size < LOAD_BATCH_MIN) { + for (OsuFile osu : batch) + load(osu); + return; + } + + try { + // create map + HashMap> map = new HashMap>(); + for (OsuFile osu : batch) { + String parent = osu.getFile().getParentFile().getName(); + String name = osu.getFile().getName(); + HashMap m = map.get(parent); + if (m == null) { + m = new HashMap(); + map.put(parent, m); + } + m.put(name, osu); + } + + // iterate through database to load OsuFiles + int count = 0; + selectAllStmt.setFetchSize(100); + ResultSet rs = selectAllStmt.executeQuery(); while (rs.next()) { - osu.beatmapID = rs.getInt(4); - osu.beatmapSetID = rs.getInt(5); - osu.title = OsuParser.getDBString(rs.getString(6)); - osu.titleUnicode = OsuParser.getDBString(rs.getString(7)); - osu.artist = OsuParser.getDBString(rs.getString(8)); - osu.artistUnicode = OsuParser.getDBString(rs.getString(9)); - osu.creator = OsuParser.getDBString(rs.getString(10)); - osu.version = OsuParser.getDBString(rs.getString(11)); - osu.source = OsuParser.getDBString(rs.getString(12)); - osu.tags = OsuParser.getDBString(rs.getString(13)); - osu.hitObjectCircle = rs.getInt(14); - osu.hitObjectSlider = rs.getInt(15); - osu.hitObjectSpinner = rs.getInt(16); - osu.HPDrainRate = rs.getFloat(17); - osu.circleSize = rs.getFloat(18); - osu.overallDifficulty = rs.getFloat(19); - osu.approachRate = rs.getFloat(20); - osu.sliderMultiplier = rs.getFloat(21); - osu.sliderTickRate = rs.getFloat(22); - osu.bpmMin = rs.getInt(23); - osu.bpmMax = rs.getInt(24); - osu.endTime = rs.getInt(25); - osu.audioFilename = new File(dir, OsuParser.getDBString(rs.getString(26))); - osu.audioLeadIn = rs.getInt(27); - osu.previewTime = rs.getInt(28); - osu.countdown = rs.getByte(29); - osu.sampleSet = OsuParser.getDBString(rs.getString(30)); - osu.stackLeniency = rs.getFloat(31); - osu.mode = rs.getByte(32); - osu.letterboxInBreaks = rs.getBoolean(33); - osu.widescreenStoryboard = rs.getBoolean(34); - osu.epilepsyWarning = rs.getBoolean(35); - osu.bg = OsuParser.getDBString(rs.getString(36)); - osu.timingPointsFromString(rs.getString(37)); - osu.breaksFromString(rs.getString(38)); - osu.comboFromString(rs.getString(39)); + String parent = rs.getString(1); + HashMap m = map.get(parent); + if (m != null) { + String name = rs.getString(2); + OsuFile osu = m.get(name); + if (osu != null) { + setOsuFileFields(rs, osu); + if (++count >= size) + break; + } + } } rs.close(); - return osu; } catch (SQLException e) { - ErrorHandler.error("Failed to get OsuFile from database.", e, true); - return null; + ErrorHandler.error("Failed to load OsuFiles from database.", e, true); } } + /** + * Sets all OsuFile fields using a given result set. + * @param rs the result set containing the fields + * @param osu the OsuFile + * @throws SQLException + */ + private static void setOsuFileFields(ResultSet rs, OsuFile osu) throws SQLException { + osu.beatmapID = rs.getInt(4); + osu.beatmapSetID = rs.getInt(5); + osu.title = OsuParser.getDBString(rs.getString(6)); + osu.titleUnicode = OsuParser.getDBString(rs.getString(7)); + osu.artist = OsuParser.getDBString(rs.getString(8)); + osu.artistUnicode = OsuParser.getDBString(rs.getString(9)); + osu.creator = OsuParser.getDBString(rs.getString(10)); + osu.version = OsuParser.getDBString(rs.getString(11)); + osu.source = OsuParser.getDBString(rs.getString(12)); + osu.tags = OsuParser.getDBString(rs.getString(13)); + osu.hitObjectCircle = rs.getInt(14); + osu.hitObjectSlider = rs.getInt(15); + osu.hitObjectSpinner = rs.getInt(16); + osu.HPDrainRate = rs.getFloat(17); + osu.circleSize = rs.getFloat(18); + osu.overallDifficulty = rs.getFloat(19); + osu.approachRate = rs.getFloat(20); + osu.sliderMultiplier = rs.getFloat(21); + osu.sliderTickRate = rs.getFloat(22); + osu.bpmMin = rs.getInt(23); + osu.bpmMax = rs.getInt(24); + osu.endTime = rs.getInt(25); + osu.audioFilename = new File(osu.getFile().getParentFile(), OsuParser.getDBString(rs.getString(26))); + osu.audioLeadIn = rs.getInt(27); + osu.previewTime = rs.getInt(28); + osu.countdown = rs.getByte(29); + osu.sampleSet = OsuParser.getDBString(rs.getString(30)); + osu.stackLeniency = rs.getFloat(31); + osu.mode = rs.getByte(32); + osu.letterboxInBreaks = rs.getBoolean(33); + osu.widescreenStoryboard = rs.getBoolean(34); + osu.epilepsyWarning = rs.getBoolean(35); + osu.bg = OsuParser.getDBString(rs.getString(36)); + osu.timingPointsFromString(rs.getString(37)); + osu.breaksFromString(rs.getString(38)); + osu.comboFromString(rs.getString(39)); + } + /** * Returns a map of file paths ({dir}/{file}) to last modified times, or * null if any error occurred. @@ -334,6 +403,7 @@ public class OsuDB { try { Map map = new HashMap(); ResultSet rs = lastModStmt.executeQuery(); + lastModStmt.setFetchSize(100); while (rs.next()) { String path = String.format("%s/%s", rs.getString(1), rs.getString(2)); long lastModified = rs.getLong(3); @@ -392,6 +462,7 @@ public class OsuDB { insertStmt.close(); lastModStmt.close(); selectStmt.close(); + selectAllStmt.close(); deleteMapStmt.close(); deleteGroupStmt.close(); connection.close();