perf: 3.4x faster planet build (2h10 -> 39 min), less RAM, deterministic output

.github/workflows/ci.yml

@@ -419,7 +419,7 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake -DTILEMAKER_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ ..
+        cmake -DTILEMAKER_BUILD_STATIC=ON -DTILEMAKER_WITH_MLT=ON -DCMAKE_BUILD_TYPE=Release -DVCPKG_TARGET_TRIPLET="${{ matrix.triplet }}" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CXX_COMPILER=g++ ..
         cmake --build . 
         strip tilemaker
 
@@ -449,6 +449,9 @@ jobs:
         run_tilemaker "${{ env.AREA }}.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}.mbtiles" --verbose --store "${{ runner.temp }}/store"
         run_tilemaker "${{ env.AREA }}-repeat.mbtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-repeat.mbtiles" --verbose --store "${{ runner.temp }}/store-repeat"
 
+        jq '.settings.tile_format = "mlt"' resources/config-openmaptiles.json > "${{ runner.temp }}/config-openmaptiles-mlt.json"
+        run_tilemaker "${{ env.AREA }}-mlt.pmtiles" "${tilemaker}" "${{ env.AREA }}.osm.pbf" --config="${{ runner.temp }}/config-openmaptiles-mlt.json" --process=resources/process-openmaptiles.lua --output="${{ env.AREA }}-mlt.pmtiles" --verbose
+
     - name: Upload generated tiles
       uses: actions/upload-artifact@v7
       with:
@@ -459,6 +462,7 @@ jobs:
           ${{ env.AREA }}.pmtiles
           ${{ env.AREA }}-repeat.mbtiles
           ${{ env.AREA }}-repeat.pmtiles
+          ${{ env.AREA }}-mlt.pmtiles
 
     - name: 'Upload compiled executable'
       uses: actions/upload-artifact@v7

CMakeLists.txt

@@ -10,6 +10,7 @@ string(STRIP "${tilemaker_version}" tilemaker_version)
 project(tilemaker VERSION ${tilemaker_version})
 
 OPTION(TILEMAKER_BUILD_STATIC "Attempt to link dependencies static" OFF)
+OPTION(TILEMAKER_WITH_MLT "Support writing MapLibre Tiles (MLT); needs CMake 3.25+" OFF)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
@@ -69,7 +70,25 @@ else()
 	include_directories(${LUAJIT_INCLUDE_DIR})
 endif()
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
+# tilemaker doesn't use coroutines, and Boost.Asio's C++20 coroutine support
+# doesn't compile with some Boost versions (e.g. 1.74 in Debian bookworm)
+add_compile_definitions(BOOST_ASIO_DISABLE_CO_AWAIT)
+
+if(TILEMAKER_WITH_MLT)
+	# The C++ MLT encoder from the MapLibre Tile spec repository. Only the submodules
+	# the encoder needs are fetched; set FETCHCONTENT_SOURCE_DIR_MLT to use a local checkout.
+	include(FetchContent)
+	set(MLT_WITH_JSON OFF CACHE BOOL "" FORCE)
+	set(MLT_WITH_TESTS OFF CACHE BOOL "" FORCE)
+	set(MLT_WITH_TOOLS OFF CACHE BOOL "" FORCE)
+	FetchContent_Declare(mlt
+		GIT_REPOSITORY https://github.com/maplibre/maplibre-tile-spec.git
+		GIT_TAG a9fa488f54ba5e7edf0151cf61f7b297d76dffe2
+		GIT_SUBMODULES cpp/vendor/fsst cpp/vendor/earcut
+		SOURCE_SUBDIR cpp)
+	FetchContent_MakeAvailable(mlt)
+endif()
 
 if(NOT TM_VERSION)
 	set(TM_VERSION v${PROJECT_VERSION})
@@ -185,8 +204,15 @@ file(GLOB tilemaker_src_files
 	src/visvalingam.cpp
 	src/way_stores.cpp
   )
+if(TILEMAKER_WITH_MLT)
+	list(APPEND tilemaker_src_files src/mlt_writer.cpp)
+endif()
 add_executable(tilemaker ${tilemaker_src_files})
 target_include_directories(tilemaker PRIVATE include)
+if(TILEMAKER_WITH_MLT)
+	target_compile_definitions(tilemaker PRIVATE TILEMAKER_WITH_MLT)
+	target_link_libraries(tilemaker mlt-cpp-encoder)
+endif()
 target_include_directories(tilemaker PRIVATE ${CMAKE_BINARY_DIR}) # for generated files
 target_link_libraries(tilemaker
 		${THREAD_LIB} ${CMAKE_DL_LIBS}

Dockerfile

@@ -15,7 +15,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     libboost-system-dev \
     luarocks \
     rapidjson-dev \
-    cmake && \
+    cmake \
+    git \
+    ca-certificates && \
     rm -rf /var/lib/apt/lists/* && \
     luarocks install luaflock
 
@@ -36,9 +38,9 @@ RUN mkdir build && \
         tm_version_arg="-DTM_VERSION=$TILEMAKER_BUILD_VERSION"; \
     fi && \
     if [ -z "$BUILD_DEBUG" ]; then \
-        cmake -DCMAKE_BUILD_TYPE=Release -DBoost_USE_DEBUG_RUNTIME=OFF $tm_version_arg ..; \
+        cmake -DCMAKE_BUILD_TYPE=Release -DBoost_USE_DEBUG_RUNTIME=OFF -DTILEMAKER_WITH_MLT=ON $tm_version_arg ..; \
     else \
-        cmake -DCMAKE_BUILD_TYPE=Debug $tm_version_arg ..; \
+        cmake -DCMAKE_BUILD_TYPE=Debug -DTILEMAKER_WITH_MLT=ON $tm_version_arg ..; \
     fi; \
     cmake --build . --parallel $(nproc) && \
     if [ -z "$BUILD_DEBUG" ]; then \

docs/CONFIGURATION.md

@@ -38,6 +38,7 @@ It also includes these global settings:
 * `bounding_box` (optional) - the bounding box to output, in [minlon, minlat, maxlon, maxlat] order
 * `default_view` (optional) - the default location for the client to view, in [lon, lat, zoom] order (MBTiles only)
 * `mvt_version` (optional) - the version of the [Mapbox Vector Tile](https://github.com/mapbox/vector-tile-spec) spec to use; defaults to 2
+* `tile_format` (optional) - `"mvt"` (default) for Mapbox Vector Tiles, or `"mlt"` for [MapLibre Tiles](https://maplibre.org/maplibre-tile-spec/). MLT output needs tilemaker to be built with MLT support (see below and INSTALL.md)
 * `combine_below` (deprecated) - whether to merge all linestrings in the tile with the same attributes: will be done at zoom levels below that specified here (e.g. `"combine_below": 14` to merge at z1-13). Please use the layer-specific parameter `combine_lines_below` instead.
 
 A typical config file would look like this:
@@ -68,6 +69,14 @@ All options are compulsory unless stated otherwise. If tilemaker baulks at the J
 
 By default tilemaker expects to find this file at config.json, but you can specify another filename with the `--config` command-line option.
 
+### MapLibre Tiles (MLT)
+
+With `"tile_format": "mlt"`, tilemaker writes [MapLibre Tiles](https://maplibre.org/maplibre-tile-spec/) instead of Mapbox Vector Tiles. Your Lua profile and layer configuration stay the same. MLT tiles are typically smaller than MVT tiles, and are supported by MapLibre GL JS and MapLibre Native. In your style, add `"encoding": "mlt"` to the vector source.
+
+* .pmtiles output is marked with the MapLibre Tile type, .mbtiles output has `format` set to `mlt`, and tiles written to a directory get the `.mlt` extension.
+* MLT stores each attribute as a typed column per layer. If the same attribute is written as an integer for some features and as a number with decimals for others in one layer, the integers are stored as doubles. Other mixed types (e.g. a number for some features and a string for others) are stored as strings.
+* `--merge` isn't supported with MLT output.
+
 ### Advanced layer configuration
 
 You can add optional parameters to layers:

docs/INSTALL.md

@@ -52,6 +52,12 @@ You can optionally use cmake to build:
     make
     sudo make install
 
+To be able to write [MapLibre Tiles](https://maplibre.org/maplibre-tile-spec/) (MLT), configure with `-DTILEMAKER_WITH_MLT=ON`. This needs cmake 3.25 or later and a C++20 compiler, and downloads the MLT encoder from GitHub during configuration (set `FETCHCONTENT_SOURCE_DIR_MLT` to use a local checkout of maplibre-tile-spec instead):
+
+    cmake -DTILEMAKER_WITH_MLT=ON ..
+
+The Makefile build doesn't include MLT support.
+
 ### Docker
 
 **The Dockerfile is not formally supported by project maintainers and you are encouraged to send pull requests to fix any issues you encounter.**

include/attribute_store.h

@@ -196,11 +196,14 @@ class AttributePairStore {
 		lookupsUncached(0)
 	{
 		// The "hot" shard has a capacity of 64K, the others are unbounded.
-		pairs.push_back(DequeMap<AttributePair>(1 << 16));
+		// reserve + emplace: DequeMap's move constructor is not noexcept, so
+		// vector growth would copy every shard.
+		pairs.reserve(ATTRIBUTE_SHARDS);
+		pairs.emplace_back(1 << 16);
 		// Reserve offset 0 as a sentinel
 		pairs[0].add(AttributePair(0, false, 0));
 		for (size_t i = 1; i < ATTRIBUTE_SHARDS; i++)
-			pairs.push_back(DequeMap<AttributePair>());
+			pairs.emplace_back();
 	}
 
 	void finalize() { finalized = true; }
@@ -345,6 +348,20 @@ struct AttributeSet {
 		return getValueAtIndex(actualIndex);
 	}
 
+	// Calls fn(pairIndex) for every pair, in the same order as getPair(0..numPairs()-1)
+	template<class F>
+	void forEachPair(F&& fn) const {
+		if (useVector) {
+			for (const uint32_t v : intValues)
+				fn(v);
+			return;
+		}
+
+		for (size_t i = 0; i < 8; i++)
+			if (isSet(i))
+				fn(getValueAtIndex(i));
+	}
+
 	AttributeSet(): useVector(false) {
 		for (int i = 0; i < sizeof(shortValues)/sizeof(shortValues[0]); i++)
 			shortValues[i] = 0;
@@ -402,6 +419,26 @@ struct AttributeSet {
 struct AttributeStore {
 	AttributeIndex add(AttributeSet &attributes);
 	std::vector<const AttributePair*> getUnsafe(AttributeIndex index) const;
+	// As getUnsafe, but calls fn(const AttributePair&) for each pair in order, without building a vector
+	template<class F>
+	void forEachPairUnsafe(AttributeIndex index, F&& fn) const {
+		forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) {
+			fn(pairStore.getPairUnsafe(pairIndex));
+		});
+	}
+	// As forEachPairUnsafe, but calls fn(pairIndex) (for pairStore.getPairUnsafe)
+	template<class F>
+	void forEachPairIndexUnsafe(AttributeIndex index, F&& fn) const {
+		const AttributeSet* attrSet;
+		try {
+			uint32_t shard = index >> (32 - SHARD_BITS);
+			uint32_t offset = index & (~(~0u << (32 - SHARD_BITS)));
+			attrSet = &sets[shard].at(offset);
+		} catch (std::out_of_range &err) {
+			throw std::runtime_error("Failed to fetch attributes at index "+std::to_string(index));
+		}
+		attrSet->forEachPair(fn);
+	}
 	void reset(); // used for testing
 	size_t size() const;
 	void reportSize() const;
@@ -411,6 +448,11 @@ struct AttributeStore {
 	void addAttribute(AttributeSet& attributeSet, std::string const &key, double v, char minzoom);
 	void addAttribute(AttributeSet& attributeSet, std::string const &key, int v, char minzoom);
 	void addAttribute(AttributeSet& attributeSet, std::string const &key, bool v, char minzoom);
+	// As above, with the key already resolved by keyStore.key2index(key) (not overloads: kaguya::LuaRef keys convert to anything)
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom);
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom);
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom);
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom);
 
 	AttributeStore():
 		finalized(false),