diff --git a/.gitignore b/.gitignore index 4f35ee2..dd556fb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ build/ dist/ /.gradle/ /bin/ +/l.msf +/switzerland-gzip.msf +/switzerland-index.msf +/switzerland-normal.msf +/switzerland.msf diff --git a/build.gradle b/build.gradle index 1a35e46..d7a36c9 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ apply plugin: 'java' apply plugin: "jacoco" apply plugin: 'eclipse' -version = '0.4.0' +version = '0.4.1' repositories { mavenCentral() @@ -72,6 +72,7 @@ dependencies { compile 'ch.poole.geo:mbtiles4j:1.2.0' compile 'org.jetbrains:annotations:15.0' compile 'com.zaxxer:SparseBitSet:1.1' + compile 'it.unimi.dsi:fastutil:8.5.8' testCompile 'junit:junit:4.12' } diff --git a/src/main/java/dev/osm/mapsplit/AbstractOsmMap.java b/src/main/java/dev/osm/mapsplit/AbstractOsmMap.java index 11b650c..d7cde5a 100644 --- a/src/main/java/dev/osm/mapsplit/AbstractOsmMap.java +++ b/src/main/java/dev/osm/mapsplit/AbstractOsmMap.java @@ -1,31 +1,33 @@ package dev.osm.mapsplit; -import static java.util.stream.Collectors.toList; - -import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.TreeSet; import org.jetbrains.annotations.NotNull; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntCollection; +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongCollection; + /*- * provides implementation aspects shared between multiple {@link OsmMap} subtypes, * particularly relating to the interpretation of the map's values. * * Structure of the values: * - * 6 4 3 2 2 2 - * 3 7 1 6 4 3 - * XXXX XXXX XXXX XXXX YYYY YYYY YYYY YYYY 1uuu uNNE nnnn nnnn nnnn nnnn nnnn nnnn + * 6 4 332 2 + * 3 7 109 7 + * XXXX XXXX XXXX XXXX YYYY YYYY YYYY YYYY 1ENN xxxx nnnn nnnn nnnn nnnn nnnn nnnn * * X - tile number * Y - tile number - * u - unused * 1 - always set to 1. This ensures that the value can be distinguished from empty positions in an array. * N - bits indicating immediate "neigbours" * E - extended "neighbour" list used - * n - bits for "short" neighbour index, in long list mode used as index + * x - additional bits used together with NN in extended list mode + * n - bits for "short" neighbour index, in extended list mode used as index * * Tiles indexed in "short" list (T original tile) * - - @@ -45,10 +47,11 @@ public abstract class AbstractOsmMap implements OsmMap { private static final long TILE_X_MASK = Const.MAX_TILE_NUMBER << TILE_X_SHIFT; private static final long TILE_Y_MASK = Const.MAX_TILE_NUMBER << TILE_Y_SHIFT; - private static final int TILE_EXT_SHIFT = 24; + private static final int TILE_EXT_SHIFT = 30; private static final long TILE_EXT_MASK = 1l << TILE_EXT_SHIFT; private static final long TILE_MARKER_MASK = 0xFFFFFFl; - private static final int NEIGHBOUR_SHIFT = TILE_EXT_SHIFT + 1; + private static final long TILE_EXT_INDEX_MASK = 0x3FFFFFFFl; + private static final int NEIGHBOUR_SHIFT = TILE_EXT_SHIFT - 2; private static final long NEIGHBOUR_MASK = 3l << NEIGHBOUR_SHIFT; private static final int ONE_BIT_SHIFT = 31; private static final long ONE_BIT_MASK = 1l << ONE_BIT_SHIFT; @@ -78,7 +81,7 @@ public int neighbour(long value) { * @see OsmMap#getAllTiles(long) */ @Override - public List getAllTiles(long key) { + public IntArrayList getAllTiles(long key) { long value = get(key); @@ -88,41 +91,41 @@ public List getAllTiles(long key) { int tx = tileX(value); int ty = tileY(value); - int neighbour = neighbour(value); - List result; + IntArrayList result; if ((value & TILE_EXT_MASK) != 0) { - int idx = (int) (value & TILE_MARKER_MASK); - result = new ArrayList<>(asList(extendedSet[idx])); + int idx = (int) (value & TILE_EXT_INDEX_MASK); + result = asList(extendedSet[idx]); + result.add(TileCoord.encode(tx, ty)); } else { - result = parseMarker(value); + result = parseNeighbourBits(value); + result.add(TileCoord.encode(tx, ty)); } + return result; + } - // TODO: some tiles (neighbour-tiles) might be double-included in the list, is this a problem?! - - // add the tile (and possible neighbours) - result.add(TileCoord.encode(tx, ty)); - if ((neighbour & OsmMap.NEIGHBOURS_EAST) != 0) { - result.add(TileCoord.encode(tx + 1, ty)); - } - if ((neighbour & OsmMap.NEIGHBOURS_SOUTH) != 0) { - result.add(TileCoord.encode(tx, ty + 1)); - } - if ((neighbour & OsmMap.NEIGHBOURS_SOUTH_EAST) == OsmMap.NEIGHBOURS_SOUTH_EAST) { - result.add(TileCoord.encode(tx + 1, ty + 1)); + /** + * Add an encoded tile to result if not already present + * + * @param tx tile x coord + * @param ty tile y coord + * @param result a List holding the tiles + */ + public void addTileIfNotPresent(int tx, int ty, @NotNull IntCollection result) { + int t = TileCoord.encode(tx, ty); + if (!result.contains(t)) { + result.add(t); } - - return result; } @Override - public void updateInt(long key, Collection tiles) { - - List longTiles = tiles.stream().map(tile -> createValue(TileCoord.decodeX(tile), TileCoord.decodeY(tile), NEIGHBOURS_NONE)).collect(toList()); - + public void updateInt(long key, IntCollection tiles) { + LongArrayList longTiles = new LongArrayList(); + for (int tile : tiles) { + longTiles.add(createValue(TileCoord.decodeX(tile), TileCoord.decodeY(tile), NEIGHBOURS_NONE)); + } this.update(key, longTiles); - } /** @@ -147,7 +150,7 @@ protected long createValue(int tileX, int tileY, int neighbours) { * @param tiles a collection of tiles to add * @return the updated value */ - protected long updateValue(long originalValue, @NotNull Collection tiles) { + protected long updateValue(long originalValue, @NotNull LongCollection tiles) { long val = originalValue; @@ -156,30 +159,20 @@ protected long updateValue(long originalValue, @NotNull Collection tiles) // neighbour list is already too large so we use the "large store" if ((val & TILE_EXT_MASK) != 0) { - int idx = (int) (val & TILE_MARKER_MASK); + int idx = (int) (val & TILE_EXT_INDEX_MASK); appendNeighbours(idx, tiles); return originalValue; } // create a expanded temp set for neighbourhood tiles - Collection expanded = new TreeSet<>(); + IntCollection expanded = new IntOpenHashSet(); for (long tile : tiles) { + final int t = (int) (tile >>> TILE_Y_SHIFT); + expanded.add(t); + int ttx = TileCoord.decodeX(t); + int tty = TileCoord.decodeY(t); - int x = tileX(tile); - int y = tileY(tile); - int neighbour = neighbour(tile); - - expanded.add(TileCoord.encode(x, y)); - - if ((neighbour & NEIGHBOURS_EAST) != 0) { - expanded.add(TileCoord.encode(x + 1, y)); - } - if ((neighbour & NEIGHBOURS_SOUTH) != 0) { - expanded.add(TileCoord.encode(x, y + 1)); - } - if (neighbour == NEIGHBOURS_SOUTH_EAST) { - expanded.add(TileCoord.encode(x + 1, y + 1)); - } + parseNnBits(expanded, neighbour(tile), ttx, tty); } // now we use the 24 reserved bits for the tiles list.. @@ -211,6 +204,27 @@ protected long updateValue(long originalValue, @NotNull Collection tiles) return val; } + /** + * Check the NN bits and add tiles to result + * + * @param result the Collection holding the result + * @param neighbour the immediate neighbours + * @param tx tile x + * @param ty tile y + */ + public void parseNnBits(IntCollection result, int neighbour, int tx, int ty) { + // add possible neighbours + if ((neighbour & OsmMap.NEIGHBOURS_EAST) != 0) { + addTileIfNotPresent(tx + 1, ty, result); + } + if ((neighbour & OsmMap.NEIGHBOURS_SOUTH) != 0) { + addTileIfNotPresent(tx, ty + 1, result); + } + if ((neighbour & OsmMap.NEIGHBOURS_SOUTH_EAST) == OsmMap.NEIGHBOURS_SOUTH_EAST) { + addTileIfNotPresent(tx + 1, ty + 1, result); + } + } + /** * Start using the list of additional tiles instead of the bits directly in the value * @@ -220,26 +234,24 @@ protected long updateValue(long originalValue, @NotNull Collection tiles) */ private long extendToNeighbourSet(long val, @NotNull Collection tiles) { - if ((val & TILE_MARKER_MASK) != 0) { - // add current stuff to tiles list - List tmpList = parseMarker(val); - for (int i : tmpList) { - long tx = TileCoord.decodeX(i); - long ty = TileCoord.decodeY(i); - long temp = tx << TILE_X_SHIFT | ty << TILE_Y_SHIFT; - - tiles.add(temp); - } + // add current stuff to tiles list + List tmpList = parseNeighbourBits(val); + for (int i : tmpList) { + long tx = TileCoord.decodeX(i); + long ty = TileCoord.decodeY(i); + long temp = tx << TILE_X_SHIFT | ty << TILE_Y_SHIFT; - // delete old marker from val - val &= ~TILE_MARKER_MASK; + tiles.add(temp); } + // delete old marker from val and old old NN values + val &= ~TILE_EXT_INDEX_MASK; + int cur = extendedBuckets++; // if we don't have enough sets, increase the array... if (cur >= extendedSet.length) { - if (extendedSet.length >= TILE_MARKER_MASK / 2) { // assumes TILE_MARKER_MASK starts at 0 + if (extendedSet.length >= TILE_EXT_INDEX_MASK / 2) { // assumes TILE_EXT_INDEX_MASK starts at 0 throw new IllegalStateException("Too many extended tile entries to expand"); } int[][] tmp = new int[2 * extendedSet.length][]; @@ -295,23 +307,27 @@ private void appendNeighbours(int index, @NotNull Collection tiles) { * @param value the encoded value * @return a list of tiles encoded in an int */ - private List parseMarker(long value) { - List result = new ArrayList<>(); + private IntArrayList parseNeighbourBits(long value) { + IntArrayList result = new IntArrayList(); + int tx = tileX(value); int ty = tileY(value); + parseNnBits(result, neighbour(value), tx, ty); - for (int i = 0; i < 24; i++) { - // if bit is not set, continue.. - if (((value >> i) & 1) == 0) { - continue; - } + if ((value & TILE_MARKER_MASK) != 0) { + for (int i = 0; i < 24; i++) { + // if bit is not set, continue.. + if (((value >> i) & 1) == 0) { + continue; + } - int v = i >= 12 ? i + 1 : i; + int v = i >= 12 ? i + 1 : i; - int tmpX = v % 5 - 2; - int tmpY = v / 5 - 2; + int tmpX = v % 5 - 2; + int tmpY = v / 5 - 2; - result.add(TileCoord.encode(tx + tmpX, ty + tmpY)); + result.add(TileCoord.encode(tx + tmpX, ty + tmpY)); + } } return result; } @@ -358,14 +374,11 @@ private static int[] merge(@NotNull int[] old, @NotNull int[] add, int len) { * @return a List of Integer */ @NotNull - private static List asList(@NotNull int[] set) { - List result = new ArrayList<>(); - + private static IntArrayList asList(@NotNull int[] set) { + IntArrayList result = new IntArrayList(); for (int i = 0; i < set.length; i++) { result.add(set[i]); } - return result; } - } diff --git a/src/main/java/dev/osm/mapsplit/ArrayMap.java b/src/main/java/dev/osm/mapsplit/ArrayMap.java index 9dada1c..5e93067 100644 --- a/src/main/java/dev/osm/mapsplit/ArrayMap.java +++ b/src/main/java/dev/osm/mapsplit/ArrayMap.java @@ -1,13 +1,14 @@ package dev.osm.mapsplit; -import org.jetbrains.annotations.NotNull; +import static java.lang.Math.min; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.stream.LongStream; -import static java.lang.Math.min; +import org.jetbrains.annotations.NotNull; + +import it.unimi.dsi.fastutil.longs.LongCollection; /** * data structure that uses huge arrays with the OSM element's ID (the key) as the array index. @@ -65,7 +66,7 @@ public final long get(long key) { } @Override - public final void update(long key, Collection tiles) { + public final void update(long key, LongCollection tiles) { long[] array = arrayForKey(key); int indexWithinArray = indexWithinArray(key); array[indexWithinArray] = updateValue(array[indexWithinArray], tiles); diff --git a/src/main/java/dev/osm/mapsplit/CommandLineParams.java b/src/main/java/dev/osm/mapsplit/CommandLineParams.java index f27dbb3..db27a03 100644 --- a/src/main/java/dev/osm/mapsplit/CommandLineParams.java +++ b/src/main/java/dev/osm/mapsplit/CommandLineParams.java @@ -88,10 +88,10 @@ final class CommandLineParams { final boolean mbTiles; /** sizes of the {@link HeapMap}s for OSM objects. Either this or {@link #maxIds} will be set. */ - final int @Nullable[] mapSizes; + final int @Nullable [] mapSizes; /** largest ID for each of the OSM object types. This will cause {@link ArrayMap}s to be used. */ - final long @Nullable[] maxIds; + final long @Nullable [] maxIds; /** maximum number of files/tiles to have open at the same time */ final int maxFiles; @@ -137,8 +137,8 @@ public CommandLineParams(@NotNull String[] args, @NotNull Logger logger) throws Option sizeOption = Option.builder(OPT_SIZE).longOpt(LONG_OPT_SIZE).hasArg().desc( "n,w,r the initial size for the node-, way- and relation maps to use (should be at least twice the number of IDs). If not supplied, defaults will be taken.") .build(); - Option maxIdsOption = Option.builder(OPT_MAX_IDS).longOpt(LONG_OPT_MAX_IDS).hasArg().desc( - "n,w,r the maximum id to allow in the node, way and relation arrays. Using this option will cause Mapsplit" + Option maxIdsOption = Option.builder(OPT_MAX_IDS).longOpt(LONG_OPT_MAX_IDS).hasArg() + .desc("n,w,r the maximum id to allow in the node, way and relation arrays. Using this option will cause Mapsplit" + " to use a different data structure that is capable of scaling to the entire planet, but will use an amount of memory proportional to the values specified here.") .build(); Option inputOption = Option.builder(OPT_INPUT).longOpt(LONG_OPT_INPUT).hasArgs().desc("a file in OSM pbf format").required().build(); @@ -233,23 +233,23 @@ public CommandLineParams(@NotNull String[] args, @NotNull Logger logger) throws maxFiles = -1; } - double border = 0.0; + double tempBorder = 0.0; if (line.hasOption(OPT_BORDER)) { String tmp = line.getOptionValue(LONG_OPT_BORDER); try { - border = Double.valueOf(tmp); - if (border < 0) { - border = 0; + tempBorder = Double.valueOf(tmp); + if (tempBorder < 0) { + tempBorder = 0; } - if (border >= 0.5) { + if (tempBorder >= 0.5) { logger.log(Level.WARNING, "border value must be less than 0.5"); - border = 0.4999; + tempBorder = 0.4999; } } catch (NumberFormatException e) { logger.log(Level.WARNING, "Could not parse border parameter, falling back to defaults"); } } - this.border = border; + this.border = tempBorder; if (line.hasOption(OPT_ZOOM)) { String tmp = line.getOptionValue(LONG_OPT_ZOOM); @@ -265,7 +265,7 @@ public CommandLineParams(@NotNull String[] args, @NotNull Logger logger) throws nodeLimit = 0; } - } catch (ParseException | NumberFormatException exp) { + } catch (ParseException | NumberFormatException exp) { // NOSONAR logger.log(Level.WARNING, exp.getMessage()); new HelpFormatter().printHelp(COMMAND_NAME, options); throw new IllegalArgumentException(exp); diff --git a/src/main/java/dev/osm/mapsplit/HeapMap.java b/src/main/java/dev/osm/mapsplit/HeapMap.java index 35a0b91..15d1543 100644 --- a/src/main/java/dev/osm/mapsplit/HeapMap.java +++ b/src/main/java/dev/osm/mapsplit/HeapMap.java @@ -1,11 +1,12 @@ package dev.osm.mapsplit; -import java.util.Collection; import java.util.stream.IntStream; import java.util.stream.LongStream; import org.jetbrains.annotations.NotNull; +import it.unimi.dsi.fastutil.longs.LongCollection; + /** * An HashMap in Heap memory, optimized for size to use in OSM * @@ -203,7 +204,7 @@ public long get(long key) { * @see OsmMap#update(long, java.util.List) */ @Override - public void update(long key, @NotNull Collection tiles) { + public void update(long key, @NotNull LongCollection tiles) { int bucket = getBucket(key); if (bucket == -1) { diff --git a/src/main/java/dev/osm/mapsplit/MapSplit.java b/src/main/java/dev/osm/mapsplit/MapSplit.java index 6211ef9..bf10990 100644 --- a/src/main/java/dev/osm/mapsplit/MapSplit.java +++ b/src/main/java/dev/osm/mapsplit/MapSplit.java @@ -31,13 +31,10 @@ import java.util.Collections; import java.util.Date; import java.util.Deque; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; -import java.util.TreeSet; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.LogManager; @@ -67,6 +64,21 @@ import ch.poole.geo.mbtiles4j.model.MetadataEntry; import crosby.binary.osmosis.OsmosisReader; import crosby.binary.osmosis.OsmosisSerializer; +import it.unimi.dsi.fastutil.bytes.Byte2ObjectMap; +import it.unimi.dsi.fastutil.bytes.Byte2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.ints.Int2ByteMap; +import it.unimi.dsi.fastutil.ints.Int2ByteOpenHashMap; +import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntCollection; +import it.unimi.dsi.fastutil.ints.IntIterable; +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongCollection; +import it.unimi.dsi.fastutil.longs.LongOpenHashSet; +import it.unimi.dsi.fastutil.longs.LongSet; public class MapSplit { @@ -111,16 +123,16 @@ public class MapSplit { // a bitset telling the algorithm which tiles need to be re-renderd private final UnsignedSparseBitSet modifiedTiles = new UnsignedSparseBitSet(); - private final Map optimizedModifiedTiles = new HashMap<>(); + private final Byte2ObjectOpenHashMap optimizedModifiedTiles = new Byte2ObjectOpenHashMap<>(); // the serializer (OSM writers) for any modified tile - private Map outFiles; + private Int2ObjectOpenHashMap outFiles; // output for mbtiles - private Map outBlobs; + private Int2ObjectOpenHashMap outBlobs; // new zoom levels for tiles during optimization - private final Map zoomMap = new HashMap<>(); + private final Int2ByteOpenHashMap zoomMap = new Int2ByteOpenHashMap(); // relations with potential forward references private final Set postProcessRelations = new HashSet<>(); @@ -162,7 +174,7 @@ public MapSplit(CommandLineParams params, Date appointmentDate) { relationMemberWayIds = new HashSet<>(); } - optimizedModifiedTiles.put(params.zoom, modifiedTiles); + optimizedModifiedTiles.put((byte) params.zoom, modifiedTiles); } /** @@ -344,8 +356,7 @@ private void checkAndFill(@NotNull Collection tiles) { tx += minX; ty += minY; - // TODO: make this a bit nicer by delegating the id-generation to the map code - int c = tx << Const.MAX_ZOOM | ty; + int c = TileCoord.encode(tx, ty); tiles.add(((long) c) << AbstractOsmMap.TILE_Y_SHIFT); modifiedTiles.set(c); } @@ -389,15 +400,15 @@ private double deltaY(double lat) { * @param neighbour bit map for neighbour tiles */ private void setModifiedTiles(int tx, int ty, int neighbour) { - modifiedTiles.set(tx << Const.MAX_ZOOM | ty); + modifiedTiles.set(TileCoord.encode(tx, ty)); if ((neighbour & OsmMap.NEIGHBOURS_EAST) != 0) { - modifiedTiles.set((tx + 1) << Const.MAX_ZOOM | ty); + modifiedTiles.set(TileCoord.encode(tx + 1, ty)); } if ((neighbour & OsmMap.NEIGHBOURS_SOUTH) != 0) { - modifiedTiles.set(tx << Const.MAX_ZOOM | (ty + 1)); + modifiedTiles.set(TileCoord.encode(tx, ty + 1)); } if (neighbour == OsmMap.NEIGHBOURS_SOUTH_EAST) { - modifiedTiles.set((tx + 1) << Const.MAX_ZOOM | (ty + 1)); + modifiedTiles.set(TileCoord.encode(tx + 1, ty + 1)); } } @@ -452,14 +463,14 @@ private void addNodeToMap(Node n, double lat, double lon) { private void addWayToMap(@NotNull Way way) { boolean modified = way.getTimestamp().after(appointmentDate); - Set tileList = new TreeSet<>(); + LongSet tileList = new LongOpenHashSet(); // mark the latest changes made to this map if (way.getTimestamp().after(latestDate)) { latestDate = way.getTimestamp(); } - List tiles = new ArrayList<>(); + LongArrayList tiles = new LongArrayList(); for (WayNode wayNode : way.getWayNodes()) { // get tileNrs for given node long tile = nmap.get(wayNode.getNodeId()); @@ -516,10 +527,8 @@ private void addWayToMap(@NotNull Way way) { * @param way the Way we are processing * @param tileList the List of tiles, encoded with {@link TileCoord} */ - private void addExtraWayToMap(@NotNull Way way, @NotNull Collection tileList) { - + private void addWayNodesToMap(@NotNull Way way, @NotNull IntCollection tileList) { for (WayNode wayNode : way.getWayNodes()) { - // update map so that the node knows about any additional // tile it has to be stored in nmap.updateInt(wayNode.getNodeId(), tileList); @@ -534,7 +543,7 @@ private void addExtraWayToMap(@NotNull Way way, @NotNull Collection til private void addRelationToMap(@NotNull Relation r) { boolean modified = r.getTimestamp().after(appointmentDate); - Collection tileList = new TreeSet<>(); + LongCollection tileList = new LongOpenHashSet(); boolean nodeWarned = false; // suppress multiple warnings about missing Nodes boolean wayWarned = false; // suppress multiple warnings about missing Ways @@ -570,7 +579,7 @@ private void addRelationToMap(@NotNull Relation r) { break; case Way: - List list = wmap.getAllTiles(m.getMemberId()); + IntArrayList list = wmap.getAllTiles(m.getMemberId()); // The referenced way is not in our data set if (list == null) { @@ -582,13 +591,12 @@ private void addRelationToMap(@NotNull Relation r) { } if (modified) { - for (Integer i : list) { + for (int i : list) { modifiedTiles.set(i); } } - // TODO: make this a bit more generic / nicer code :/ - for (Integer i : list) { + for (int i : list) { tileList.add(((long) i) << AbstractOsmMap.TILE_Y_SHIFT); } break; @@ -612,8 +620,8 @@ private void addRelationToMap(@NotNull Relation r) { } } - for (Integer i : list) { - tileList.add(((long) i) << HeapMap.TILE_Y_SHIFT); + for (int i : list) { + tileList.add(((long) i) << AbstractOsmMap.TILE_Y_SHIFT); } break; default: @@ -629,7 +637,7 @@ private void addRelationToMap(@NotNull Relation r) { // no need to fill tile list here as that will have already happened for any element with geometry - long val = tileList.iterator().next(); + long val = tileList.iterator().nextLong(); int tx = rmap.tileX(val); int ty = rmap.tileY(val); @@ -806,8 +814,8 @@ public void process(EntityContainer ec) { if (ec instanceof WayContainer) { Way w = ((WayContainer) ec).getEntity(); if (relationMemberWayIds.contains(w.getId())) { - List tileList = wmap.getAllTiles(w.getId()); - addExtraWayToMap(w, tileList); + IntArrayList tileList = wmap.getAllTiles(w.getId()); + addWayNodesToMap(w, tileList); } } } @@ -864,17 +872,17 @@ private void optimize(final int nodeLimit) { // at high zoom levels this will contains // lots of Nodes that are in more than // one tile - Map stats = new HashMap<>(); + Int2IntOpenHashMap stats = new Int2IntOpenHashMap(); nmap.keys().forEach((long k) -> { - List tiles = nmap.getAllTiles(k); + IntArrayList tiles = nmap.getAllTiles(k); if (tiles != null) { - for (Integer t : tiles) { - Integer count = stats.get(t); - if (count != null) { + for (int t : tiles) { + if (stats.containsKey(t)) { + int count = stats.get(t); count++; - stats.put(t, count); + stats.replace(t, count); } else { - stats.put(t, 1); + stats.putIfAbsent(t, 1); } } } else { @@ -882,9 +890,9 @@ private void optimize(final int nodeLimit) { } }); long nodeCount = 0; - List keys = new ArrayList<>(stats.keySet()); + IntArrayList keys = new IntArrayList(stats.keySet()); Collections.sort(keys); - for (Integer key : keys) { + for (int key : keys) { int value = stats.get(key); nodeCount += value; if (!zoomMap.containsKey(key)) { // not mapped @@ -917,9 +925,9 @@ private void optimize(final int nodeLimit) { } } } - for (Entry optimzedTile : zoomMap.entrySet()) { - int idx = optimzedTile.getKey(); - int newTileZoom = optimzedTile.getValue(); + for (Int2ByteMap.Entry optimzedTile : zoomMap.int2ByteEntrySet()) { + int idx = optimzedTile.getIntKey(); + byte newTileZoom = optimzedTile.getByteValue(); modifiedTiles.clear(idx); idx = mapToNewTile(idx, newTileZoom); UnsignedSparseBitSet tileSet = optimizedModifiedTiles.get(newTileZoom); @@ -935,7 +943,7 @@ private void optimize(final int nodeLimit) { } } - class CountResult { + private class CountResult { int total; int[] keys; Integer[] counts; @@ -949,15 +957,15 @@ class CountResult { * @param stats a map containing the per tile stats * @return a CountResult object */ - CountResult getCounts(int idx, int zoomDiff, @NotNull Map stats) { + private CountResult getCounts(int idx, int zoomDiff, @NotNull Map stats) { // determine the counts for the other tiles in the zoomed out tile - int x0 = ((idx >>> Const.MAX_ZOOM) >> zoomDiff) << zoomDiff; - int y0 = ((idx & (int) Const.MAX_TILE_NUMBER) >> zoomDiff) << zoomDiff; + int x0 = (TileCoord.decodeX(idx) >> zoomDiff) << zoomDiff; + int y0 = (TileCoord.decodeY(idx) >> zoomDiff) << zoomDiff; int side = 2 << (zoomDiff - 1); int[] keys = new int[side * side]; for (int i = 0; i < side; i++) { for (int j = 0; j < side; j++) { - keys[i * side + j] = ((x0 + i) << Const.MAX_ZOOM) | (y0 + j); + keys[i * side + j] = TileCoord.encode(x0 + i, y0 + j); } } Integer[] counts = new Integer[keys.length]; @@ -983,9 +991,9 @@ CountResult getCounts(int idx, int zoomDiff, @NotNull Map stat * @return packed tile id at newTileZoom */ private int mapToNewTile(int idx, int newTileZoom) { - int xNew = (idx >>> Const.MAX_ZOOM) >> (params.zoom - newTileZoom); - int yNew = (idx & (int) Const.MAX_TILE_NUMBER) >> (params.zoom - newTileZoom); - return xNew << Const.MAX_ZOOM | yNew; + int xNew = TileCoord.decodeX(idx) >> (params.zoom - newTileZoom); + int yNew = TileCoord.decodeY(idx) >> (params.zoom - newTileZoom); + return TileCoord.encode(xNew, yNew); } /** @@ -1133,8 +1141,8 @@ public void clipPoly(@NotNull File polygonFile) throws IOException { break; } - int tx = idx >>> Const.MAX_ZOOM; - int ty = (int) (idx & Const.MAX_TILE_NUMBER); + int tx = TileCoord.decodeX(idx); + int ty = TileCoord.decodeY(idx); boolean in = isInside(tx, ty, inside, outside); @@ -1166,9 +1174,9 @@ public void store(@NotNull String basename, boolean metadata, boolean mbTiles) t } Bound bounds = null; int minZoom = params.zoom; - for (Entry omt : optimizedModifiedTiles.entrySet()) { + for (Byte2ObjectMap.Entry omt : optimizedModifiedTiles.byte2ObjectEntrySet()) { final UnsignedSparseBitSet tileSet = omt.getValue(); - final int currentZoom = omt.getKey(); + final int currentZoom = omt.getByteKey(); if (currentZoom < minZoom) { minZoom = currentZoom; } @@ -1184,9 +1192,9 @@ public void store(@NotNull String basename, boolean metadata, boolean mbTiles) t while (true) { complete = false; - outFiles = new HashMap<>(); + outFiles = new Int2ObjectOpenHashMap<>(); if (mbTiles) { - outBlobs = new HashMap<>(); + outBlobs = new Int2ObjectOpenHashMap<>(); } // Setup out-files... @@ -1200,8 +1208,8 @@ public void store(@NotNull String basename, boolean metadata, boolean mbTiles) t if (outFiles.get(idx) == null) { - int tileX = idx >>> Const.MAX_ZOOM; - int tileY = (int) (idx & Const.MAX_TILE_NUMBER); + int tileX = TileCoord.decodeX(idx); + int tileY = TileCoord.decodeY(idx); OutputStream target = null; if (mbTiles) { @@ -1251,8 +1259,8 @@ public void store(@NotNull String basename, boolean metadata, boolean mbTiles) t class BoundSink implements Sink { - Bound overallBounds = null; - Set mappedTiles = new HashSet<>(); + Bound overallBounds = null; + IntOpenHashSet mappedTiles = new IntOpenHashSet(); /** * Get the overall bounds of the data @@ -1272,7 +1280,7 @@ public void complete() { public void process(EntityContainer ec) { long id = ec.getEntity().getId(); - Iterable tiles; + IntIterable tiles; if (ec instanceof NodeContainer) { tiles = nmap.getAllTiles(id); @@ -1306,11 +1314,10 @@ public void process(EntityContainer ec) { for (int i : tiles) { // map original zoom tiles to optimized ones // and remove duplicates - Byte newZoom = zoomMap.get(i); - if (newZoom != null) { + byte newZoom = (byte) params.zoom; + if (zoomMap.containsKey(i)) { + newZoom = zoomMap.get(i); i = mapToNewTile(i, newZoom); - } else { - newZoom = (byte) params.zoom; } if (currentZoom == newZoom) { mappedTiles.add(i); @@ -1343,23 +1350,29 @@ public void close() { BoundSink sink = new BoundSink(); reader.setSink(sink); + long processingStart = System.currentTimeMillis(); runThread(new Thread(reader)); + if (params.timing) { + LOGGER.log(Level.INFO, "Processing tile block took {0} ms", System.currentTimeMillis() - processingStart); + } if (!complete) { throw new IOException("Could not fully read file in storing run"); } + long writingStart = System.currentTimeMillis(); // Finish and close files... - for (Entry entry : outFiles.entrySet()) { + for (Int2ObjectMap.Entry entry : outFiles.int2ObjectEntrySet()) { OsmosisSerializer ser = entry.getValue(); ser.complete(); ser.flush(); ser.close(); if (mbTiles) { - int tileX = entry.getKey() >>> Const.MAX_ZOOM; - int tileY = (int) (entry.getKey() & Const.MAX_TILE_NUMBER); + final int key = entry.getIntKey(); + int tileX = key >>> Const.MAX_ZOOM; + int tileY = (int) (key & Const.MAX_TILE_NUMBER); int y = ymax - tileY - 1; // TMS scheme - ByteArrayOutputStream blob = outBlobs.get(entry.getKey()); + ByteArrayOutputStream blob = outBlobs.get(key); try { w.addTile(blob.toByteArray(), currentZoom, tileX, y); } catch (MBTilesWriteException e) { // NOSONAR @@ -1369,10 +1382,14 @@ public void close() { } } + if (params.timing) { + LOGGER.log(Level.INFO, "Writing tile block took {0} ms", System.currentTimeMillis() - writingStart); + } if (params.verbose) { LOGGER.log(Level.INFO, "Wrote {0} tiles, continuing with next block of tiles", outFiles.size()); } - // remove mappings form this pass + + // remove mappings from this pass outFiles.clear(); if (mbTiles) { outBlobs.clear(); diff --git a/src/main/java/dev/osm/mapsplit/OsmMap.java b/src/main/java/dev/osm/mapsplit/OsmMap.java index ae2876c..5fc3437 100644 --- a/src/main/java/dev/osm/mapsplit/OsmMap.java +++ b/src/main/java/dev/osm/mapsplit/OsmMap.java @@ -13,9 +13,12 @@ */ import java.util.Collection; -import java.util.List; import java.util.stream.LongStream; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntCollection; +import it.unimi.dsi.fastutil.longs.LongCollection; + /** * This is the central data structure of Mapsplit. It maps OSM element IDs to the tile(s) each element is located in. As * there are three types of OSM elements (node/way/relation), there will be three instances of this kind of data @@ -64,7 +67,7 @@ public interface OsmMap { * @param key the entries key (ID) * @param tiles a Collection of tiles in encoded form including neighbour bits */ - public abstract void update(long key, Collection tiles); + public abstract void update(long key, LongCollection tiles); /** * variant of {@link #update(long, Collection)} that takes individual tile coords (encoded with TileCoord) without @@ -73,7 +76,7 @@ public interface OsmMap { * @param key the entries key (ID) * @param tiles a Collection of tiles in encoded form without neighbour bits */ - public abstract void updateInt(long key, Collection tiles); + public abstract void updateInt(long key, IntCollection tiles); /** * returns a list of all tiles this key is in. This contains the base-tile, neighbours and tiles where this key is @@ -82,7 +85,7 @@ public interface OsmMap { * @param key the node, way or relation we're looking at * @return a list of all tiles where this key is used */ - public abstract List getAllTiles(long key); + public abstract IntArrayList getAllTiles(long key); /** * for debugging this method tells you how much of the buckets are used. A load < 0,5 is desirable for good speed diff --git a/src/test/java/dev/osm/mapsplit/OsmMapTest.java b/src/test/java/dev/osm/mapsplit/OsmMapTest.java index fb6f761..d9a02ed 100644 --- a/src/test/java/dev/osm/mapsplit/OsmMapTest.java +++ b/src/test/java/dev/osm/mapsplit/OsmMapTest.java @@ -14,6 +14,9 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongList; + @RunWith(Parameterized.class) public class OsmMapTest { @@ -26,7 +29,7 @@ public OsmMapTest(Supplier mapSupplier) { @Parameterized.Parameters public static Collection> testees() { - return List.of(() -> new ArrayMap(100), () -> new HeapMap(100)); + return List.of(() -> new ArrayMap(100), () -> new HeapMap(100)); } @Test @@ -39,7 +42,7 @@ public void testPutUpdateAndRetrieve() { /* put some entries into the tile and its eastern neighbor */ - map.put(42, tileX , tileY, OsmMap.NEIGHBOURS_NONE); + map.put(42, tileX, tileY, OsmMap.NEIGHBOURS_NONE); map.put(0, tileX, tileY, OsmMap.NEIGHBOURS_EAST); map.put(100, tileX + 1, tileY, OsmMap.NEIGHBOURS_NONE); @@ -54,9 +57,7 @@ public void testPutUpdateAndRetrieve() { /* update one of the values and check again */ - map.update(42, List.of( - map.createValue(tileX + 1, tileY, OsmMap.NEIGHBOURS_NONE), - map.createValue(tileX, tileY + 1, OsmMap.NEIGHBOURS_NONE))); + map.update(42, LongList.of(map.createValue(tileX + 1, tileY, OsmMap.NEIGHBOURS_NONE), map.createValue(tileX, tileY + 1, OsmMap.NEIGHBOURS_NONE))); assertEquals(3, map.getAllTiles(42).size()); @@ -88,7 +89,7 @@ public void testTile00() { assertEquals(0, map.tileY(value1)); assertEquals(4, map.getAllTiles(1).size()); - map.update(2, List.of(map.createValue(0, 0, OsmMap.NEIGHBOURS_NONE))); + map.update(2, LongList.of(map.createValue(0, 0, OsmMap.NEIGHBOURS_NONE))); assertTrue(map.getAllTiles(0).contains(0 << 16 | 0)); } @@ -106,7 +107,7 @@ public void testLongTileLists() { listOfTiles.add(map.createValue(10000 + i, 500, OsmMap.NEIGHBOURS_NONE)); map.put(42, 10000, 500, OsmMap.NEIGHBOURS_NONE); - map.update(42, new ArrayList<>(listOfTiles)); + map.update(42, new LongArrayList(listOfTiles)); assertEquals(1 + i, map.getAllTiles(42).size()); diff --git a/test-data/.gitignore b/test-data/.gitignore index 57c4bb1..a631cfc 100644 --- a/test-data/.gitignore +++ b/test-data/.gitignore @@ -1,3 +1,11 @@ /iraq-latest.osm.pbf /sachsen-latest.osm.pbf /switzerland-latest.osm.pbf +/austria-latest.osm.pbf +/austria.msf +/switzerland-exact.osm.pbf +/switzerland-padded.osm.pbf +/test1.msf +/test.msf +/turkmenistan-latest.osm.pbf +/zh.pbf