/* * opsu! - an open-source osu! client * Copyright (C) 2014, 2015 Jeffrey Han * * opsu! is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * opsu! is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with opsu!. If not, see . */ package itdelatrisu.opsu.beatmap; import itdelatrisu.opsu.ErrorHandler; import itdelatrisu.opsu.Options; import itdelatrisu.opsu.Utils; import itdelatrisu.opsu.db.BeatmapDB; import itdelatrisu.opsu.io.MD5InputStreamWrapper; import java.io.*; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.newdawn.slick.Color; import org.newdawn.slick.util.Log; /** * Parser for beatmaps. */ public class BeatmapParser { /** The string lookup database. */ private static HashMap stringdb = new HashMap(); /** The expected pattern for beatmap directories, used to find beatmap set IDs. */ private static final String DIR_MSID_PATTERN = "^\\d+ .*"; /** The current file being parsed. */ private static File currentFile; /** The current directory number while parsing. */ private static int currentDirectoryIndex = -1; /** The total number of directories to parse. */ private static int totalDirectories = -1; /** Parser statuses. */ public enum Status { NONE, PARSING, CACHE, INSERTING }; /** The current status. */ private static Status status = Status.NONE; /** If no Provider supports a MessageDigestSpi implementation for the MD5 algorithm. */ private static boolean hasNoMD5Algorithm = false; // This class should not be instantiated. private BeatmapParser() {} /** * Invokes parser for each OSU file in a root directory and * adds the beatmaps to a new BeatmapSetList. * @param root the root directory (search has depth 1) */ public static void parseAllFiles(File root) { // create a new BeatmapSetList BeatmapSetList.create(); // create a new watch service if (Options.isWatchServiceEnabled()) BeatmapWatchService.create(); // parse all directories parseDirectories(root.listFiles()); } /** * Invokes parser for each directory in the given array and * adds the beatmaps to the existing BeatmapSetList. * @param dirs the array of directories to parse * @return the last BeatmapSetNode parsed, or null if none */ public static BeatmapSetNode parseDirectories(File[] dirs) { if (dirs == null) return null; // progress tracking status = Status.PARSING; currentDirectoryIndex = 0; totalDirectories = dirs.length; // get last modified map from database Map map = BeatmapDB.getLastModifiedMap(); // beatmap lists List> allBeatmaps = new LinkedList>(); List cachedBeatmaps = new LinkedList(); // loaded from database List parsedBeatmaps = new LinkedList(); // loaded from parser // watch service BeatmapWatchService ws = (Options.isWatchServiceEnabled()) ? BeatmapWatchService.get() : null; // parse directories BeatmapSetNode lastNode = null; long timestamp = System.currentTimeMillis(); for (File dir : dirs) { currentDirectoryIndex++; if (!dir.isDirectory()) continue; // find all OSU files File[] files = dir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.toLowerCase().endsWith(".osu"); } }); if (files == null || files.length < 1) continue; // create a new group entry ArrayList beatmaps = new ArrayList(); for (File file : files) { currentFile = file; // check if beatmap is cached String path = String.format("%s/%s", dir.getName(), file.getName()); if (map != null) { Long lastModified = map.get(path); if (lastModified != null) { // check last modified times if (lastModified == file.lastModified()) { // add to cached beatmap list Beatmap beatmap = new Beatmap(file); beatmaps.add(beatmap); cachedBeatmaps.add(beatmap); continue; } else BeatmapDB.delete(dir.getName(), file.getName()); } } // Parse hit objects only when needed to save time/memory. // Change boolean to 'true' to parse them immediately. Beatmap beatmap = null; try { beatmap = parseFile(file, dir, beatmaps, false); } catch(Exception e) { Log.error("could not parse beatmap " + file.getName() + ": " + e.getMessage()); // TODO: show right bottom notification } // add to parsed beatmap list if (beatmap != null) { beatmap.dateAdded = timestamp; beatmaps.add(beatmap); parsedBeatmaps.add(beatmap); } } // add group entry if non-empty if (!beatmaps.isEmpty()) { beatmaps.trimToSize(); allBeatmaps.add(beatmaps); if (ws != null) ws.registerAll(dir.toPath()); } // stop parsing files (interrupted) if (Thread.interrupted()) break; } // load cached entries from database if (!cachedBeatmaps.isEmpty()) { status = Status.CACHE; // Load array fields only when needed to save time/memory. // Change flag to 'LOAD_ALL' to load them immediately. BeatmapDB.load(cachedBeatmaps, BeatmapDB.LOAD_NONARRAY); } // add group entries to BeatmapSetList for (ArrayList beatmaps : allBeatmaps) { Collections.sort(beatmaps); lastNode = BeatmapSetList.get().addSongGroup(beatmaps); } // clear string DB stringdb = new HashMap(); // add beatmap entries to database if (!parsedBeatmaps.isEmpty()) { status = Status.INSERTING; BeatmapDB.insert(parsedBeatmaps); } status = Status.NONE; currentFile = null; currentDirectoryIndex = -1; totalDirectories = -1; return lastNode; } public static void parseOnlyTimingPoints(Beatmap map) { if (map == null || map.getFile() == null || !map.getFile().exists()) { return; } if (map.timingPoints == null) { map.timingPoints = new ArrayList(); } try ( InputStream bis = new BufferedInputStream(new FileInputStream(map.getFile())); MD5InputStreamWrapper md5stream = (!hasNoMD5Algorithm) ? new MD5InputStreamWrapper(bis) : null; BufferedReader in = new BufferedReader(new InputStreamReader((md5stream != null) ? md5stream : bis, "UTF-8")); ) { String line; boolean found = false; while((line = in.readLine()) != null) { line = line.trim(); if(!isValidLine(line)) { continue; } if ("[TimingPoints]".equals(line)) { found = true; continue; } if (found) { if (line.startsWith("[")) { break; } parseSectionTimingPoints(map, line); } } map.timingPoints.trimToSize(); } catch (IOException e) { ErrorHandler.error(String.format("Failed to read file '%s'.", map.getFile().getAbsolutePath()), e, false); } catch (NoSuchAlgorithmException e) { ErrorHandler.error("Failed to get MD5 hash stream.", e, true); // retry without MD5 hasNoMD5Algorithm = true; parseOnlyTimingPoints(map); } } private static void parseSectionTimingPoints(Beatmap beatmap, String line) { TimingPoint timingPoint = new TimingPoint(line); if(!timingPoint.isInherited()) { int bpm = Math.round(60000 / timingPoint.getBeatLength()); if( beatmap.bpmMin == 0 ) { beatmap.bpmMin = beatmap.bpmMax = bpm; } else if( bpm < beatmap.bpmMin ) { beatmap.bpmMin = bpm; } else if( bpm > beatmap.bpmMax ) { beatmap.bpmMax = bpm; } } beatmap.timingPoints.add(timingPoint); } /** * Parses a beatmap. * @param file the file to parse * @param dir the directory containing the beatmap * @param beatmaps the song group * @param parseObjects if true, hit objects will be fully parsed now * @return the new beatmap */ private static Beatmap parseFile(File file, File dir, ArrayList beatmaps, boolean parseObjects) { Beatmap beatmap = new Beatmap(file); beatmap.timingPoints = new ArrayList(); try ( InputStream bis = new BufferedInputStream(new FileInputStream(file)); MD5InputStreamWrapper md5stream = (!hasNoMD5Algorithm) ? new MD5InputStreamWrapper(bis) : null; BufferedReader in = new BufferedReader(new InputStreamReader((md5stream != null) ? md5stream : bis, "UTF-8")); ) { String line = in.readLine(); String tokens[] = null; while (line != null) { line = line.trim(); if (!isValidLine(line)) { line = in.readLine(); continue; } switch (line) { case "[General]": while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; if ((tokens = tokenize(line)) == null) continue; try { switch (tokens[0]) { case "AudioFilename": File audioFileName = new File(dir, tokens[1]); if (!beatmaps.isEmpty()) { // if possible, reuse the same File object from another Beatmap in the group File groupAudioFileName = beatmaps.get(0).audioFilename; if (groupAudioFileName != null && tokens[1].equalsIgnoreCase(groupAudioFileName.getName())) audioFileName = groupAudioFileName; } if (!audioFileName.isFile()) { // try to find the file with a case-insensitive match boolean match = false; for (String s : dir.list()) { if (s.equalsIgnoreCase(tokens[1])) { audioFileName = new File(dir, s); match = true; break; } } if (!match) { Log.error(String.format("Audio file '%s' not found in directory '%s'.", tokens[1], dir.getName())); return null; } } beatmap.audioFilename = audioFileName; break; case "AudioLeadIn": beatmap.audioLeadIn = Integer.parseInt(tokens[1]); break; // case "AudioHash": // deprecated // beatmap.audioHash = tokens[1]; // break; case "PreviewTime": beatmap.previewTime = Integer.parseInt(tokens[1]); break; case "Countdown": beatmap.countdown = Byte.parseByte(tokens[1]); break; case "SampleSet": beatmap.sampleSet = getDBString(tokens[1]); break; case "StackLeniency": beatmap.stackLeniency = Float.parseFloat(tokens[1]); break; case "Mode": beatmap.mode = Byte.parseByte(tokens[1]); /* Non-Opsu! standard files not implemented (obviously). */ if (beatmap.mode != Beatmap.MODE_OSU) return null; break; case "LetterboxInBreaks": beatmap.letterboxInBreaks = Utils.parseBoolean(tokens[1]); break; case "WidescreenStoryboard": beatmap.widescreenStoryboard = Utils.parseBoolean(tokens[1]); break; case "EpilepsyWarning": beatmap.epilepsyWarning = Utils.parseBoolean(tokens[1]); default: break; } } catch (Exception e) { Log.warn(String.format("Failed to read line '%s' for file '%s'.", line, file.getAbsolutePath()), e); } } break; case "[Editor]": while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; /* Not implemented. */ // if ((tokens = tokenize(line)) == null) // continue; // try { // switch (tokens[0]) { // case "Bookmarks": // String[] bookmarks = tokens[1].split(","); // beatmap.bookmarks = new int[bookmarks.length]; // for (int i = 0; i < bookmarks.length; i++) // osu.bookmarks[i] = Integer.parseInt(bookmarks[i]); // break; // case "DistanceSpacing": // beatmap.distanceSpacing = Float.parseFloat(tokens[1]); // break; // case "BeatDivisor": // beatmap.beatDivisor = Byte.parseByte(tokens[1]); // break; // case "GridSize": // beatmap.gridSize = Integer.parseInt(tokens[1]); // break; // case "TimelineZoom": // beatmap.timelineZoom = Integer.parseInt(tokens[1]); // break; // default: // break; // } // } catch (Exception e) { // Log.warn(String.format("Failed to read editor line '%s' for file '%s'.", // line, file.getAbsolutePath()), e); // } } break; case "[Metadata]": while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; if ((tokens = tokenize(line)) == null) continue; try { switch (tokens[0]) { case "Title": beatmap.title = getDBString(tokens[1]); break; case "TitleUnicode": beatmap.titleUnicode = getDBString(tokens[1]); break; case "Artist": beatmap.artist = getDBString(tokens[1]); break; case "ArtistUnicode": beatmap.artistUnicode = getDBString(tokens[1]); break; case "Creator": beatmap.creator = getDBString(tokens[1]); break; case "Version": beatmap.version = getDBString(tokens[1]); break; case "Source": beatmap.source = getDBString(tokens[1]); break; case "Tags": beatmap.tags = getDBString(tokens[1].toLowerCase()); break; case "BeatmapID": beatmap.beatmapID = Integer.parseInt(tokens[1]); break; case "BeatmapSetID": beatmap.beatmapSetID = Integer.parseInt(tokens[1]); break; } } catch (Exception e) { Log.warn(String.format("Failed to read metadata '%s' for file '%s'.", line, file.getAbsolutePath()), e); } if (beatmap.beatmapSetID <= 0) { // try to determine MSID from directory name if (dir != null && dir.isDirectory()) { String dirName = dir.getName(); if (!dirName.isEmpty() && dirName.matches(DIR_MSID_PATTERN)) beatmap.beatmapSetID = Integer.parseInt(dirName.substring(0, dirName.indexOf(' '))); } } } break; case "[Difficulty]": while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; if ((tokens = tokenize(line)) == null) continue; try { switch (tokens[0]) { case "HPDrainRate": beatmap.HPDrainRate = Float.parseFloat(tokens[1]); break; case "CircleSize": beatmap.circleSize = Float.parseFloat(tokens[1]); break; case "OverallDifficulty": beatmap.overallDifficulty = Float.parseFloat(tokens[1]); break; case "ApproachRate": beatmap.approachRate = Float.parseFloat(tokens[1]); break; case "SliderMultiplier": beatmap.sliderMultiplier = Float.parseFloat(tokens[1]); break; case "SliderTickRate": beatmap.sliderTickRate = Float.parseFloat(tokens[1]); break; } } catch (Exception e) { Log.warn(String.format("Failed to read difficulty '%s' for file '%s'.", line, file.getAbsolutePath()), e); } } if (beatmap.approachRate == -1f) // not in old format beatmap.approachRate = beatmap.overallDifficulty; break; case "[Events]": while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; tokens = line.split(","); switch (tokens[0]) { case "0": // background tokens[2] = tokens[2].replaceAll("^\"|\"$", ""); String ext = BeatmapParser.getExtension(tokens[2]); if (ext.equals("jpg") || ext.equals("png")) beatmap.bg = new File(dir, getDBString(tokens[2])); break; case "2": // break periods try { if (beatmap.breaks == null) // optional, create if needed beatmap.breaks = new ArrayList(); beatmap.breaks.add(Integer.parseInt(tokens[1])); beatmap.breaks.add(Integer.parseInt(tokens[2])); } catch (Exception e) { Log.warn(String.format("Failed to read break period '%s' for file '%s'.", line, file.getAbsolutePath()), e); } break; default: /* Not implemented. */ break; } } if (beatmap.breaks != null) beatmap.breaks.trimToSize(); break; case "[TimingPoints]": while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; try { parseTimingPoint(beatmap, line); } catch (Exception e) { Log.warn(String.format("Failed to read timing point '%s' for file '%s'.", line, file.getAbsolutePath()), e); } } beatmap.timingPoints.trimToSize(); break; case "[Colours]": LinkedList colors = new LinkedList(); while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; if ((tokens = tokenize(line)) == null) continue; try { String[] rgb = tokens[1].split(","); Color color = new Color( Integer.parseInt(rgb[0]), Integer.parseInt(rgb[1]), Integer.parseInt(rgb[2]) ); switch (tokens[0]) { case "Combo1": case "Combo2": case "Combo3": case "Combo4": case "Combo5": case "Combo6": case "Combo7": case "Combo8": colors.add(color); break; case "SliderBorder": beatmap.sliderBorder = color; break; default: break; } } catch (Exception e) { Log.warn(String.format("Failed to read color '%s' for file '%s'.", line, file.getAbsolutePath()), e); } } if (!colors.isEmpty()) beatmap.combo = colors.toArray(new Color[colors.size()]); break; case "[HitObjects]": int type = 0; while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; /* Only type counts parsed at this time. */ tokens = line.split(","); try { type = Integer.parseInt(tokens[3]); if ((type & HitObject.TYPE_CIRCLE) > 0) beatmap.hitObjectCircle++; else if ((type & HitObject.TYPE_SLIDER) > 0) beatmap.hitObjectSlider++; else //if ((type & HitObject.TYPE_SPINNER) > 0) beatmap.hitObjectSpinner++; } catch (Exception e) { Log.warn(String.format("Failed to read hit object '%s' for file '%s'.", line, file.getAbsolutePath()), e); } } try { // map length = last object end time (TODO: end on slider?) if ((type & HitObject.TYPE_SPINNER) > 0) { // some 'endTime' fields contain a ':' character (?) int index = tokens[5].indexOf(':'); if (index != -1) tokens[5] = tokens[5].substring(0, index); beatmap.endTime = Integer.parseInt(tokens[5]); } else if (type != 0) beatmap.endTime = Integer.parseInt(tokens[2]); } catch (Exception e) { Log.warn(String.format("Failed to read hit object end time '%s' for file '%s'.", line, file.getAbsolutePath()), e); } break; default: line = in.readLine(); break; } } if (md5stream != null) beatmap.md5Hash = md5stream.getMD5(); } catch (IOException e) { ErrorHandler.error(String.format("Failed to read file '%s'.", file.getAbsolutePath()), e, false); } catch (NoSuchAlgorithmException e) { ErrorHandler.error("Failed to get MD5 hash stream.", e, true); // retry without MD5 hasNoMD5Algorithm = true; return parseFile(file, dir, beatmaps, parseObjects); } // no associated audio file? if (beatmap.audioFilename == null) return null; // parse hit objects now? if (parseObjects) parseHitObjects(beatmap); return beatmap; } /** * Parses a timing point and adds it to the beatmap. * @param beatmap the beatmap * @param line the line containing the unparsed timing point */ private static void parseTimingPoint(Beatmap beatmap, String line) { // parse timing point TimingPoint timingPoint = new TimingPoint(line); beatmap.timingPoints.add(timingPoint); // calculate BPM if (!timingPoint.isInherited()) { int bpm = Math.round(60000 / timingPoint.getBeatLength()); if (beatmap.bpmMin == 0) { beatmap.bpmMin = beatmap.bpmMax = bpm; } else if (bpm < beatmap.bpmMin) { beatmap.bpmMin = bpm; } else if (bpm > beatmap.bpmMax) { beatmap.bpmMax = bpm; } } } /** * Parses all timing points in a beatmap. * @param beatmap the beatmap to parse */ public static void parseTimingPoints(Beatmap beatmap) { if (beatmap.timingPoints != null) // already parsed return; beatmap.timingPoints = new ArrayList(); try (BufferedReader in = new BufferedReader(new FileReader(beatmap.getFile()))) { String line = in.readLine(); while (line != null) { line = line.trim(); if (!line.equals("[TimingPoints]")) line = in.readLine(); else break; } if (line == null) // no timing points return; while ((line = in.readLine()) != null) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; try { parseTimingPoint(beatmap, line); } catch (Exception e) { Log.warn(String.format("Failed to read timing point '%s' for file '%s'.", line, beatmap.getFile().getAbsolutePath()), e); } } beatmap.timingPoints.trimToSize(); } catch (IOException e) { ErrorHandler.error(String.format("Failed to read file '%s'.", beatmap.getFile().getAbsolutePath()), e, false); } } /** * Parses all hit objects in a beatmap. * @param beatmap the beatmap to parse */ public static void parseHitObjects(Beatmap beatmap) { if (beatmap.objects != null) // already parsed return; beatmap.objects = new HitObject[(beatmap.hitObjectCircle + beatmap.hitObjectSlider + beatmap.hitObjectSpinner)]; try (BufferedReader in = new BufferedReader(new FileReader(beatmap.getFile()))) { String line = in.readLine(); while (line != null) { line = line.trim(); if (!line.equals("[HitObjects]")) line = in.readLine(); else break; } if (line == null) { Log.warn(String.format("No hit objects found in Beatmap '%s'.", beatmap.toString())); return; } // combo info Color[] combo = beatmap.getComboColors(); int comboIndex = 0; // color index int comboNumber = 1; // combo number int objectIndex = 0; boolean first = true; while ((line = in.readLine()) != null && objectIndex < beatmap.objects.length) { line = line.trim(); if (!isValidLine(line)) continue; if (line.charAt(0) == '[') break; // lines must have at minimum 5 parameters int tokenCount = line.length() - line.replace(",", "").length(); if (tokenCount < 4) continue; try { // create a new HitObject for each line HitObject hitObject = new HitObject(line); // set combo info // - new combo: get next combo index, reset combo number // - else: maintain combo index, increase combo number if (hitObject.isNewCombo() || first) { int skip = (hitObject.isSpinner() ? 0 : 1) + hitObject.getComboSkip(); for (int i = 0; i < skip; i++) { comboIndex = (comboIndex + 1) % combo.length; comboNumber = 1; } first = false; } hitObject.setComboIndex(comboIndex); hitObject.setComboNumber(comboNumber++); beatmap.objects[objectIndex++] = hitObject; } catch (Exception e) { Log.warn(String.format("Failed to read hit object '%s' for beatmap '%s'.", line, beatmap.toString()), e); } } // check that all objects were parsed if (objectIndex != beatmap.objects.length) ErrorHandler.error(String.format("Parsed %d objects for beatmap '%s', %d objects expected.", objectIndex, beatmap.toString(), beatmap.objects.length), null, true); } catch (IOException e) { ErrorHandler.error(String.format("Failed to read file '%s'.", beatmap.getFile().getAbsolutePath()), e, false); } } /** * Returns false if the line is too short or commented. */ private static boolean isValidLine(String line) { return (line.length() > 1 && !line.startsWith("//")); } /** * Splits line into two strings: tag, value. * If no ':' character is present, null will be returned. */ private static String[] tokenize(String line) { int index = line.indexOf(':'); if (index == -1) { Log.debug(String.format("Failed to tokenize line: '%s'.", line)); return null; } String[] tokens = new String[2]; tokens[0] = line.substring(0, index).trim(); tokens[1] = line.substring(index + 1).trim(); return tokens; } /** * Returns the file extension of a file. * @param file the file name */ public static String getExtension(String file) { int i = file.lastIndexOf('.'); return (i != -1) ? file.substring(i + 1).toLowerCase() : ""; } /** * Returns the name of the current file being parsed, or null if none. */ public static String getCurrentFileName() { if (status == Status.PARSING) return (currentFile != null) ? currentFile.getName() : null; else return (status == Status.NONE) ? null : ""; } /** * Returns the progress of file parsing, or -1 if not parsing. * @return the completion percent [0, 100] or -1 */ public static int getParserProgress() { if (currentDirectoryIndex == -1 || totalDirectories == -1) return -1; return currentDirectoryIndex * 100 / totalDirectories; } /** * Returns the current parser status. */ public static Status getStatus() { return status; } /** * Returns the String object in the database for the given String. * If none, insert the String into the database and return the original String. * @param s the string to retrieve * @return the string object */ public static String getDBString(String s) { String DBString = stringdb.get(s); if (DBString == null) { stringdb.put(s, s); return s; } else return DBString; } }