Changed cached OsuFile loading.

Instead of being loaded individually from the database, OsuFiles are now loaded in batch by traversing the database.  In most cases, this should cause a ~10% loading speed improvement.  Defaults to the previous approach if few OsuFiles are being loaded.

Other changes:
- Don't drop indexes upon batch insertion if few entries are being inserted.
- Trying to increase fetch size with setFetchSize(100).
- Added more parser statuses.

Signed-off-by: Jeffrey Han <itdelatrisu@gmail.com>
This commit is contained in:
Jeffrey Han 2015-03-07 23:24:19 -05:00
parent c6041d8cba
commit 40a800eb76
3 changed files with 172 additions and 75 deletions

View File

@ -31,6 +31,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.newdawn.slick.Color;
@ -55,8 +56,11 @@ public class OsuParser {
/** The total number of directories to parse. */
private static int totalDirectories = -1;
/** Whether or not the database is currently being updated. */
private static boolean updatingDatabase = false;
/** Parser statuses. */
public enum Status { NONE, PARSING, CACHE, INSERTING };
/** The current status. */
private static Status status = Status.NONE;
// This class should not be instantiated.
private OsuParser() {}
@ -85,14 +89,19 @@ public class OsuParser {
return null;
// progress tracking
status = Status.PARSING;
currentDirectoryIndex = 0;
totalDirectories = dirs.length;
// get last modified map from database
Map<String, Long> map = OsuDB.getLastModifiedMap();
// OsuFile lists
List<ArrayList<OsuFile>> allOsuFiles = new LinkedList<ArrayList<OsuFile>>();
List<OsuFile> cachedOsuFiles = new LinkedList<OsuFile>(); // loaded from database
List<OsuFile> parsedOsuFiles = new LinkedList<OsuFile>(); // loaded from parser
// parse directories
LinkedList<OsuFile> parsedOsuFiles = new LinkedList<OsuFile>();
OsuGroupNode lastNode = null;
for (File dir : dirs) {
currentDirectoryIndex++;
@ -120,7 +129,10 @@ public class OsuParser {
// check last modified times
long lastModified = map.get(path);
if (lastModified == file.lastModified()) {
osuFiles.add(OsuDB.getOsuFile(dir, file));
// add to cached beatmap list
OsuFile osu = new OsuFile(file);
osuFiles.add(osu);
cachedOsuFiles.add(osu);
continue;
} else
OsuDB.delete(dir.getName(), file.getName());
@ -130,13 +142,17 @@ public class OsuParser {
// Change boolean to 'true' to parse them immediately.
OsuFile osu = parseFile(file, dir, osuFiles, false);
if (osu != null)
// add to parsed beatmap list
if (osu != null) {
osuFiles.add(osu);
parsedOsuFiles.add(osu);
}
}
if (!osuFiles.isEmpty()) { // add entry if non-empty
// add group entry if non-empty
if (!osuFiles.isEmpty()) {
osuFiles.trimToSize();
Collections.sort(osuFiles);
lastNode = OsuGroupList.get().addSongGroup(osuFiles);
allOsuFiles.add(osuFiles);
}
// stop parsing files (interrupted)
@ -144,16 +160,28 @@ public class OsuParser {
break;
}
// load cached entries from database
if (!cachedOsuFiles.isEmpty()) {
status = Status.CACHE;
OsuDB.load(cachedOsuFiles);
}
// add group entries to OsuGroupList
for (ArrayList<OsuFile> osuFiles : allOsuFiles) {
Collections.sort(osuFiles);
lastNode = OsuGroupList.get().addSongGroup(osuFiles);
}
// clear string DB
stringdb = new HashMap<String, String>();
// add entries to database
// add beatmap entries to database
if (!parsedOsuFiles.isEmpty()) {
updatingDatabase = true;
status = Status.INSERTING;
OsuDB.insert(parsedOsuFiles);
updatingDatabase = false;
}
status = Status.NONE;
currentFile = null;
currentDirectoryIndex = -1;
totalDirectories = -1;
@ -197,7 +225,8 @@ public class OsuParser {
if (!osuFiles.isEmpty()) {
// if possible, reuse the same File object from another OsuFile in the group
File groupAudioFileName = osuFiles.get(0).audioFilename;
if (tokens[1].equalsIgnoreCase(groupAudioFileName.getName()))
if (groupAudioFileName != null &&
tokens[1].equalsIgnoreCase(groupAudioFileName.getName()))
audioFileName = groupAudioFileName;
}
if (!audioFileName.isFile()) {
@ -550,10 +579,6 @@ public class OsuParser {
if (parseObjects)
parseHitObjects(osu);
// add OsuFile to song group
if (osuFiles != null)
osuFiles.add(osu);
return osu;
}
@ -666,10 +691,10 @@ public class OsuParser {
* Returns the name of the current file being parsed, or null if none.
*/
public static String getCurrentFileName() {
if (updatingDatabase)
return "";
else
if (status == Status.PARSING)
return (currentFile != null) ? currentFile.getName() : null;
else
return (status == Status.NONE) ? null : "";
}
/**
@ -684,9 +709,9 @@ public class OsuParser {
}
/**
* Returns whether or not the beatmap database is currently being updated.
* Returns the current parser status.
*/
public static boolean isUpdatingDatabase() { return updatingDatabase; }
public static Status getStatus() { return status; }
/**
* Returns the String object in the database for the given String.

View File

@ -409,7 +409,8 @@ public class UI {
text = "Unpacking new beatmaps...";
progress = OszUnpacker.getUnpackerProgress();
} else if ((file = OsuParser.getCurrentFileName()) != null) {
text = (OsuParser.isUpdatingDatabase()) ? "Updating database..." : "Loading beatmaps...";
text = (OsuParser.getStatus() == OsuParser.Status.INSERTING) ?
"Updating database..." : "Loading beatmaps...";
progress = OsuParser.getParserProgress();
} else if ((file = SoundController.getCurrentFileName()) != null) {
text = "Loading sounds...";

View File

@ -43,11 +43,17 @@ public class OsuDB {
*/
private static final String DATABASE_VERSION = "2014-03-04";
/** Minimum batch size to invoke batch loading. */
private static final int LOAD_BATCH_MIN = 100;
/** Minimum batch size to invoke batch insertion. */
private static final int INSERT_BATCH_MIN = 100;
/** Database connection. */
private static Connection connection;
/** Query statements. */
private static PreparedStatement insertStmt, selectStmt, lastModStmt, deleteMapStmt, deleteGroupStmt;
private static PreparedStatement insertStmt, selectStmt, selectAllStmt, lastModStmt, deleteMapStmt, deleteGroupStmt;
// This class should not be instantiated.
private OsuDB() {}
@ -75,6 +81,7 @@ public class OsuDB {
"?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
);
lastModStmt = connection.prepareStatement("SELECT dir, file, lastModified FROM beatmaps");
selectAllStmt = connection.prepareStatement("SELECT * FROM beatmaps");
selectStmt = connection.prepareStatement("SELECT * FROM beatmaps WHERE dir = ? AND file = ?");
deleteMapStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ? AND file = ?");
deleteGroupStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ?");
@ -191,8 +198,11 @@ public class OsuDB {
connection.setAutoCommit(false);
// drop indexes
String sql = "DROP INDEX IF EXISTS idx";
stmt.executeUpdate(sql);
boolean recreateIndexes = (batch.size() >= INSERT_BATCH_MIN);
if (recreateIndexes) {
String sql = "DROP INDEX IF EXISTS idx";
stmt.executeUpdate(sql);
}
// batch insert
for (OsuFile osu : batch) {
@ -202,8 +212,10 @@ public class OsuDB {
insertStmt.executeBatch();
// re-create indexes
sql = "CREATE INDEX idx ON beatmaps (dir, file)";
stmt.executeUpdate(sql);
if (recreateIndexes) {
String sql = "CREATE INDEX idx ON beatmaps (dir, file)";
stmt.executeUpdate(sql);
}
// restore previous auto-commit mode
connection.commit();
@ -263,66 +275,123 @@ public class OsuDB {
}
/**
* Returns an OsuFile from the database, or null if any error occurred.
* @param dir the directory
* @param file the file
* @return the OsuFile with all fields filled, or null if any error occurred
* Loads OsuFile fields from the database.
* @param osu the OsuFile object
*/
public static OsuFile getOsuFile(File dir, File file) {
public static void load(OsuFile osu) {
if (connection == null)
return null;
return;
try {
OsuFile osu = new OsuFile(file);
selectStmt.setString(1, dir.getName());
selectStmt.setString(2, file.getName());
selectStmt.setString(1, osu.getFile().getParentFile().getName());
selectStmt.setString(2, osu.getFile().getName());
ResultSet rs = selectStmt.executeQuery();
if (rs.next())
setOsuFileFields(rs, osu);
rs.close();
} catch (SQLException e) {
ErrorHandler.error("Failed to load OsuFile from database.", e, true);
}
}
/**
* Loads OsuFile fields from the database in a batch.
* @param batch a list of OsuFile objects
*/
public static void load(List<OsuFile> batch) {
if (connection == null)
return;
// batch size too small
int size = batch.size();
if (size < LOAD_BATCH_MIN) {
for (OsuFile osu : batch)
load(osu);
return;
}
try {
// create map
HashMap<String, HashMap<String, OsuFile>> map = new HashMap<String, HashMap<String, OsuFile>>();
for (OsuFile osu : batch) {
String parent = osu.getFile().getParentFile().getName();
String name = osu.getFile().getName();
HashMap<String, OsuFile> m = map.get(parent);
if (m == null) {
m = new HashMap<String, OsuFile>();
map.put(parent, m);
}
m.put(name, osu);
}
// iterate through database to load OsuFiles
int count = 0;
selectAllStmt.setFetchSize(100);
ResultSet rs = selectAllStmt.executeQuery();
while (rs.next()) {
osu.beatmapID = rs.getInt(4);
osu.beatmapSetID = rs.getInt(5);
osu.title = OsuParser.getDBString(rs.getString(6));
osu.titleUnicode = OsuParser.getDBString(rs.getString(7));
osu.artist = OsuParser.getDBString(rs.getString(8));
osu.artistUnicode = OsuParser.getDBString(rs.getString(9));
osu.creator = OsuParser.getDBString(rs.getString(10));
osu.version = OsuParser.getDBString(rs.getString(11));
osu.source = OsuParser.getDBString(rs.getString(12));
osu.tags = OsuParser.getDBString(rs.getString(13));
osu.hitObjectCircle = rs.getInt(14);
osu.hitObjectSlider = rs.getInt(15);
osu.hitObjectSpinner = rs.getInt(16);
osu.HPDrainRate = rs.getFloat(17);
osu.circleSize = rs.getFloat(18);
osu.overallDifficulty = rs.getFloat(19);
osu.approachRate = rs.getFloat(20);
osu.sliderMultiplier = rs.getFloat(21);
osu.sliderTickRate = rs.getFloat(22);
osu.bpmMin = rs.getInt(23);
osu.bpmMax = rs.getInt(24);
osu.endTime = rs.getInt(25);
osu.audioFilename = new File(dir, OsuParser.getDBString(rs.getString(26)));
osu.audioLeadIn = rs.getInt(27);
osu.previewTime = rs.getInt(28);
osu.countdown = rs.getByte(29);
osu.sampleSet = OsuParser.getDBString(rs.getString(30));
osu.stackLeniency = rs.getFloat(31);
osu.mode = rs.getByte(32);
osu.letterboxInBreaks = rs.getBoolean(33);
osu.widescreenStoryboard = rs.getBoolean(34);
osu.epilepsyWarning = rs.getBoolean(35);
osu.bg = OsuParser.getDBString(rs.getString(36));
osu.timingPointsFromString(rs.getString(37));
osu.breaksFromString(rs.getString(38));
osu.comboFromString(rs.getString(39));
String parent = rs.getString(1);
HashMap<String, OsuFile> m = map.get(parent);
if (m != null) {
String name = rs.getString(2);
OsuFile osu = m.get(name);
if (osu != null) {
setOsuFileFields(rs, osu);
if (++count >= size)
break;
}
}
}
rs.close();
return osu;
} catch (SQLException e) {
ErrorHandler.error("Failed to get OsuFile from database.", e, true);
return null;
ErrorHandler.error("Failed to load OsuFiles from database.", e, true);
}
}
/**
* Sets all OsuFile fields using a given result set.
* @param rs the result set containing the fields
* @param osu the OsuFile
* @throws SQLException
*/
private static void setOsuFileFields(ResultSet rs, OsuFile osu) throws SQLException {
osu.beatmapID = rs.getInt(4);
osu.beatmapSetID = rs.getInt(5);
osu.title = OsuParser.getDBString(rs.getString(6));
osu.titleUnicode = OsuParser.getDBString(rs.getString(7));
osu.artist = OsuParser.getDBString(rs.getString(8));
osu.artistUnicode = OsuParser.getDBString(rs.getString(9));
osu.creator = OsuParser.getDBString(rs.getString(10));
osu.version = OsuParser.getDBString(rs.getString(11));
osu.source = OsuParser.getDBString(rs.getString(12));
osu.tags = OsuParser.getDBString(rs.getString(13));
osu.hitObjectCircle = rs.getInt(14);
osu.hitObjectSlider = rs.getInt(15);
osu.hitObjectSpinner = rs.getInt(16);
osu.HPDrainRate = rs.getFloat(17);
osu.circleSize = rs.getFloat(18);
osu.overallDifficulty = rs.getFloat(19);
osu.approachRate = rs.getFloat(20);
osu.sliderMultiplier = rs.getFloat(21);
osu.sliderTickRate = rs.getFloat(22);
osu.bpmMin = rs.getInt(23);
osu.bpmMax = rs.getInt(24);
osu.endTime = rs.getInt(25);
osu.audioFilename = new File(osu.getFile().getParentFile(), OsuParser.getDBString(rs.getString(26)));
osu.audioLeadIn = rs.getInt(27);
osu.previewTime = rs.getInt(28);
osu.countdown = rs.getByte(29);
osu.sampleSet = OsuParser.getDBString(rs.getString(30));
osu.stackLeniency = rs.getFloat(31);
osu.mode = rs.getByte(32);
osu.letterboxInBreaks = rs.getBoolean(33);
osu.widescreenStoryboard = rs.getBoolean(34);
osu.epilepsyWarning = rs.getBoolean(35);
osu.bg = OsuParser.getDBString(rs.getString(36));
osu.timingPointsFromString(rs.getString(37));
osu.breaksFromString(rs.getString(38));
osu.comboFromString(rs.getString(39));
}
/**
* Returns a map of file paths ({dir}/{file}) to last modified times, or
* null if any error occurred.
@ -334,6 +403,7 @@ public class OsuDB {
try {
Map<String, Long> map = new HashMap<String, Long>();
ResultSet rs = lastModStmt.executeQuery();
lastModStmt.setFetchSize(100);
while (rs.next()) {
String path = String.format("%s/%s", rs.getString(1), rs.getString(2));
long lastModified = rs.getLong(3);
@ -392,6 +462,7 @@ public class OsuDB {
insertStmt.close();
lastModStmt.close();
selectStmt.close();
selectAllStmt.close();
deleteMapStmt.close();
deleteGroupStmt.close();
connection.close();