Add support for node processing in second stage

This is completely analog to how way processing happens in the second
stage. So from the select_relation_member() function in the Lua config
file you can now also return a list of nodes that you request to process
again.

Note that this will only re-process the member nodes themselves, ways
which have these nodes as members are not re-processed. Also there is no
way to mark member nodes of ways, only member nodes of relations. So
this will allow processing, say, stop positions in public transport route
relations but not, say, barriers on roads.

This change is now possible because we removed support for the old
middle format which didn't allow storing the complete nodes (with tags).

Author: joto

flex-config/README.md

@@ -17,6 +17,8 @@ following order (from easiest to understand to the more complex ones):
 
 After that you can dive into more advanced topics:
 
+* [public-transport.lua](public-transport.lua) -- Use multi-stage processing
+  to bring tags from public transport relations to member nodes and ways
 * [route-relations.lua](route-relations.lua) -- Use multi-stage processing
   to bring tags from relations to member ways
 * [unitable.lua](unitable.lua) -- Put all OSM data into a single table

flex-config/public-transport.lua

@@ -0,0 +1,224 @@
+-- This config example file is released into the Public Domain.
+
+-- This file shows how to use multi-stage processing to bring tags from
+-- public transport relations into member nodes and ways. This allows
+-- advanced processing of public transport networks including stops.
+
+-- Nodes tagged as public transport stops are imported into the 'stops' table,
+-- if they are part of a public transport relation. Ways tagged as highway or
+-- railway or imported into the 'lines' table. The public transport routes
+-- themselves will be in the 'routes' table, but without any geometry. As a
+-- "bonus" public transport stop area relations will be imported into the
+-- 'stop_areas' table.
+--
+-- For the 'stops' and 'lines' table two-stage processing is used. The
+-- 'rel_refs' text column will contain a list of all ref tags found in parent
+-- relations with type=route and route=public_transport. The 'rel_ids' column
+-- will be an integer array containing the relation ids. These could be used,
+-- for instance, to look up other relation tags from the 'routes' table.
+
+local tables = {}
+
+tables.stops = osm2pgsql.define_node_table('stops', {
+    { column = 'tags',     type = 'jsonb' },
+    { column = 'rel_refs', type = 'text' }, -- for the refs from the relations
+    { column = 'rel_ids',  sql_type = 'int8[]' }, -- array with integers (for relation IDs)
+    { column = 'geom',     type = 'point', not_null = true },
+})
+
+tables.lines = osm2pgsql.define_way_table('lines', {
+    { column = 'tags',     type = 'jsonb' },
+    { column = 'rel_refs', type = 'text' }, -- for the refs from the relations
+    { column = 'rel_ids',  sql_type = 'int8[]' }, -- array with integers (for relation IDs)
+    { column = 'geom',     type = 'linestring', not_null = true },
+})
+
+-- Tables don't have to have a geometry column
+tables.routes = osm2pgsql.define_relation_table('routes', {
+    { column = 'ref',  type = 'text' },
+    { column = 'type', type = 'text' },
+    { column = 'from', type = 'text' },
+    { column = 'to',   type = 'text' },
+    { column = 'tags', type = 'jsonb' },
+})
+
+-- Stop areas contain everything belonging to a specific public transport
+-- stop. We model them here by adding a center point as geometry plus the
+-- radius of a circle that contains everything in that stop.
+tables.stop_areas = osm2pgsql.define_relation_table('stop_areas', {
+    { column = 'tags',   type = 'jsonb' },
+    { column = 'radius', type = 'real', not_null = true },
+    { column = 'geom',   type = 'point', not_null = true },
+})
+
+-- This will be used to store information about relations queryable by member
+-- node/way id. These are table of tables. The outer table is indexed by the
+-- node/way id, the inner table indexed by the relation id. This way even if
+-- the information about a relation is added twice, it will be in there only
+-- once. It is always good to write your osm2pgsql Lua code in an idempotent
+-- way, i.e. it can be called any number of times and will lead to the same
+-- result.
+local n2r = {}
+local w2r = {}
+
+local function clean_tags(tags)
+    tags.odbl = nil
+    tags.created_by = nil
+    tags.source = nil
+    tags['source:ref'] = nil
+
+    return next(tags) == nil
+end
+
+local function unique_array(array)
+    local result = {}
+
+    local last = nil
+    for _, v in ipairs(array) do
+        if v ~= last then
+            result[#result + 1] = v
+            last = v
+        end
+    end
+
+    return result
+end
+
+local separator = '·' -- use middle dot as separator character
+
+local function add_rel_data(row, d)
+    if not d then
+        return
+    end
+
+    local refs = {}
+    local ids = {}
+    for rel_id, rel_ref in pairs(d) do
+        refs[#refs + 1] = rel_ref
+        ids[#ids + 1] = rel_id
+    end
+    table.sort(refs)
+    table.sort(ids)
+
+    row.rel_refs = table.concat(unique_array(refs), separator)
+    row.rel_ids = '{' .. table.concat(unique_array(ids), ',') .. '}'
+end
+
+function osm2pgsql.process_node(object)
+    -- We are only interested in public transport stops here, and they are
+    -- only available in the second stage.
+    if osm2pgsql.stage ~= 2 then
+        return
+    end
+
+    local row = {
+        tags = object.tags,
+        geom = object:as_point()
+    }
+
+    -- If there is any data from parent relations, add it in
+    add_rel_data(row, n2r[object.id])
+
+    tables.stops:insert(row)
+end
+
+function osm2pgsql.process_way(object)
+    -- We are only interested in highways and railways
+    if not object.tags.highway and not object.tags.railway then
+        return
+    end
+
+    clean_tags(object.tags)
+
+    -- Data we will store in the 'lines' table always has the tags from
+    -- the way
+    local row = {
+        tags = object.tags,
+        geom = object:as_linestring()
+    }
+
+    -- If there is any data from parent relations, add it in
+    add_rel_data(row, w2r[object.id])
+
+    tables.lines:insert(row)
+end
+
+local pt = {
+    bus = true,
+    light_rail = true,
+    subway = true,
+    tram = true,
+    trolleybus = true,
+}
+
+-- We are only interested in certain route relations with a ref tag
+local function wanted_relation(tags)
+    return tags.type == 'route' and pt[tags.route] and tags.ref
+end
+
+-- This function is called for every added, modified, or deleted relation.
+-- Its only job is to return the ids of all member nodes/ways of the specified
+-- relation we want to see in stage 2 again. It MUST NOT store any information
+-- about the relation!
+function osm2pgsql.select_relation_members(relation)
+    -- Only interested in public transport relations with refs
+    if wanted_relation(relation.tags) then
+        local node_ids = {}
+        local way_ids = {}
+
+        for _, member in ipairs(relation.members) do
+            if member.type == 'n' and member.role == 'stop' then
+                node_ids[#node_ids + 1] = member.ref
+            elseif member.type == 'w' and member.role == '' then
+                way_ids[#way_ids + 1] = member.ref
+            end
+        end
+
+        return {
+            nodes = node_ids,
+            ways = way_ids,
+        }
+    end
+end
+
+-- The process_relation() function should store all information about relation
+-- members that might be needed in stage 2.
+function osm2pgsql.process_relation(object)
+    if object.tags.type == 'public_transport' and object.tags.public_transport == 'stop_area' then
+        local x1, y1, x2, y2 = object:as_geometrycollection():transform(3857):get_bbox()
+        local radius = math.sqrt((x2-x1)*(x2-x1) + (y2-y1)*(y2-y1))
+        tables.stop_areas:insert({
+            tags = object.tags,
+            geom = object:as_geometrycollection():centroid(),
+            radius = radius,
+        })
+        return
+    end
+
+    if wanted_relation(object.tags) then
+        tables.routes:insert({
+            type = object.tags.route,
+            ref = object.tags.ref,
+            from = object.tags.from,
+            to = object.tags.to,
+            tags = object.tags,
+        })
+
+        -- Go through all the members and store relation ids and refs so they
+        -- can be found by the member node/way id.
+        for _, member in ipairs(object.members) do
+            if member.type == 'n' then
+                if not n2r[member.ref] then
+                    n2r[member.ref] = {}
+                end
+                n2r[member.ref][object.id] = object.tags.ref
+            elseif member.type == 'w' then
+                if not w2r[member.ref] then
+                    w2r[member.ref] = {}
+                end
+                w2r[member.ref][object.id] = object.tags.ref
+            end
+        end
+    end
+end
+

src/init.lua

@@ -46,6 +46,16 @@ function osm2pgsql.define_area_table(_name, _columns, _options)
     return _define_table_impl('area', _name, _columns, _options)
 end
 
+function osm2pgsql.node_member_ids(relation)
+    local ids = {}
+    for _, member in ipairs(relation.members) do
+        if member.type == 'n' then
+            ids[#ids + 1] = member.ref
+        end
+    end
+    return ids
+end
+
 function osm2pgsql.way_member_ids(relation)
     local ids = {}
     for _, member in ipairs(relation.members) do

src/middle-pgsql.cpp

@@ -691,7 +691,10 @@ INSERT INTO osm2pgsql_changed_relations
         m_db_connection.exec(build_sql(*m_options, query));
     }
 
-    load_id_list(m_db_connection, "osm2pgsql_changed_ways", parent_ways);
+    if (parent_ways) {
+        load_id_list(m_db_connection, "osm2pgsql_changed_ways", parent_ways);
+    }
+
     load_id_list(m_db_connection, "osm2pgsql_changed_relations",
                  parent_relations);
 
@@ -700,9 +703,17 @@ INSERT INTO osm2pgsql_changed_relations
     timer.stop();
 
     log_debug("Found {} new/changed nodes in input.", changed_nodes.size());
-    log_debug("  Found in {} their {} parent ways and {} parent relations.",
-              std::chrono::duration_cast<std::chrono::seconds>(timer.elapsed()),
-              parent_ways->size(), parent_relations->size());
+
+    auto const elapsed_sec =
+        std::chrono::duration_cast<std::chrono::seconds>(timer.elapsed());
+
+    if (parent_ways) {
+        log_debug("  Found in {} their {} parent ways and {} parent relations.",
+                  elapsed_sec, parent_ways->size(), parent_relations->size());
+    } else {
+        log_debug("  Found in {} their {} parent relations.", elapsed_sec,
+                  parent_relations->size());
+    }
 }
 
 void middle_pgsql_t::get_way_parents(idlist_t const &changed_ways,
@@ -771,6 +782,21 @@ void middle_pgsql_t::way_set(osmium::Way const &way)
 
 namespace {
 
+/**
+ * Build node in buffer from database results.
+ */
+void build_node(osmid_t id, pg_result_t const &res, int res_num, int offset,
+                osmium::memory::Buffer *buffer, bool with_attributes)
+{
+    osmium::builder::NodeBuilder builder{*buffer};
+    builder.set_id(id);
+
+    if (with_attributes) {
+        set_attributes_on_builder(&builder, res, res_num, offset);
+    }
+    pgsql_parse_json_tags(res.get_value(res_num, offset + 1), buffer, &builder);
+}
+
 /**
  * Build way in buffer from database results.
  */
@@ -789,6 +815,24 @@ void build_way(osmid_t id, pg_result_t const &res, int res_num, int offset,
 
 } // anonymous namespace
 
+bool middle_query_pgsql_t::node_get(osmid_t id,
+                                    osmium::memory::Buffer *buffer) const
+{
+    assert(buffer);
+
+    auto const res = m_db_connection.exec_prepared("get_node", id);
+
+    if (res.num_tuples() != 1) {
+        return false;
+    }
+
+    build_node(id, res, 0, 0, buffer, m_store_options.with_attributes);
+
+    buffer->commit();
+
+    return true;
+}
+
 bool middle_query_pgsql_t::way_get(osmid_t id,
                                    osmium::memory::Buffer *buffer) const
 {

src/middle-pgsql.hpp

@@ -59,6 +59,8 @@ class middle_query_pgsql_t : public middle_query_t
 
     size_t nodes_get_list(osmium::WayNodeList *nodes) const override;
 
+    bool node_get(osmid_t id, osmium::memory::Buffer *buffer) const override;
+
     bool way_get(osmid_t id, osmium::memory::Buffer *buffer) const override;
 
     size_t rel_members_get(osmium::Relation const &rel,

src/middle-ram.cpp

@@ -278,6 +278,16 @@ std::size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const
     return count;
 }
 
+bool middle_ram_t::node_get(osmid_t id, osmium::memory::Buffer *buffer) const
+{
+    assert(buffer);
+
+    if (m_store_options.nodes) {
+        return get_object(osmium::item_type::node, id, buffer);
+    }
+    return false;
+}
+
 bool middle_ram_t::way_get(osmid_t id, osmium::memory::Buffer *buffer) const
 {
     assert(buffer);

src/middle-ram.hpp

@@ -68,6 +68,8 @@ class middle_ram_t : public middle_t, public middle_query_t
 
     std::size_t nodes_get_list(osmium::WayNodeList *nodes) const override;
 
+    bool node_get(osmid_t id, osmium::memory::Buffer *buffer) const override;
+
     bool way_get(osmid_t id, osmium::memory::Buffer *buffer) const override;
 
     size_t rel_members_get(osmium::Relation const &rel,