Skip to content

Commit 0aff9ef

Browse files
bitnerbitner
authored
fix for upsert_item and upsert_items functions (#232)
1 parent e44ec16 commit 0aff9ef

File tree

9 files changed

+4921
-30
lines changed

9 files changed

+4921
-30
lines changed

‎src/pgstac/migrations/pgstac.0.8.2-unreleased.sql‎

Lines changed: 361 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,361 @@
1+
SET client_min_messages TO WARNING;
2+
SET SEARCH_PATH to pgstac, public;
3+
RESET ROLE;
4+
DO $$
5+
DECLARE
6+
BEGIN
7+
IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN
8+
CREATE EXTENSION IF NOT EXISTS postgis;
9+
END IF;
10+
IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN
11+
CREATE EXTENSION IF NOT EXISTS btree_gist;
12+
END IF;
13+
END;
14+
$$ LANGUAGE PLPGSQL;
15+
16+
DO $$
17+
BEGIN
18+
CREATE ROLE pgstac_admin;
19+
EXCEPTION WHEN duplicate_object THEN
20+
RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
21+
END
22+
$$;
23+
24+
DO $$
25+
BEGIN
26+
CREATE ROLE pgstac_read;
27+
EXCEPTION WHEN duplicate_object THEN
28+
RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
29+
END
30+
$$;
31+
32+
DO $$
33+
BEGIN
34+
CREATE ROLE pgstac_ingest;
35+
EXCEPTION WHEN duplicate_object THEN
36+
RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
37+
END
38+
$$;
39+
40+
41+
GRANT pgstac_admin TO current_user;
42+
43+
-- Function to make sure pgstac_admin is the owner of items
44+
CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$
45+
DECLARE
46+
f RECORD;
47+
BEGIN
48+
FOR f IN (
49+
SELECT
50+
concat(
51+
oid::regproc::text,
52+
'(',
53+
coalesce(pg_get_function_identity_arguments(oid),''),
54+
')'
55+
) AS name,
56+
CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ
57+
FROM pg_proc
58+
WHERE
59+
pronamespace=to_regnamespace('pgstac')
60+
AND proowner != to_regrole('pgstac_admin')
61+
AND proname NOT LIKE 'pg_stat%'
62+
)
63+
LOOP
64+
BEGIN
65+
EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name);
66+
EXCEPTION WHEN others THEN
67+
RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
68+
END;
69+
END LOOP;
70+
FOR f IN (
71+
SELECT
72+
oid::regclass::text as name,
73+
CASE relkind
74+
WHEN 'i' THEN 'INDEX'
75+
WHEN 'I' THEN 'INDEX'
76+
WHEN 'p' THEN 'TABLE'
77+
WHEN 'r' THEN 'TABLE'
78+
WHEN 'v' THEN 'VIEW'
79+
WHEN 'S' THEN 'SEQUENCE'
80+
ELSE NULL
81+
END as typ
82+
FROM pg_class
83+
WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat'
84+
)
85+
LOOP
86+
BEGIN
87+
EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name);
88+
EXCEPTION WHEN others THEN
89+
RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
90+
END;
91+
END LOOP;
92+
RETURN;
93+
END;
94+
$$ LANGUAGE PLPGSQL;
95+
SELECT pgstac_admin_owns();
96+
97+
CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin;
98+
99+
GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin;
100+
GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin;
101+
GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin;
102+
103+
ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public;
104+
ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public;
105+
ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public;
106+
107+
GRANT USAGE ON SCHEMA pgstac to pgstac_read;
108+
ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read;
109+
ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read;
110+
ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read;
111+
112+
GRANT pgstac_read TO pgstac_ingest;
113+
GRANT ALL ON SCHEMA pgstac TO pgstac_ingest;
114+
ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest;
115+
ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest;
116+
117+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read;
118+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read;
119+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read;
120+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest;
121+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest;
122+
123+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read;
124+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read;
125+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read;
126+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest;
127+
ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest;
128+
129+
SET SEARCH_PATH TO pgstac, public;
130+
SET ROLE pgstac_admin;
131+
132+
DO $$
133+
BEGIN
134+
DROP FUNCTION IF EXISTS analyze_items;
135+
EXCEPTION WHEN others THEN
136+
RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
137+
END
138+
$$;
139+
DO $$
140+
BEGIN
141+
DROP FUNCTION IF EXISTS validate_constraints;
142+
EXCEPTION WHEN others THEN
143+
RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
144+
END
145+
$$;
146+
147+
-- Install these idempotently as migrations do not put them before trying to modify the collections table
148+
149+
150+
CREATE OR REPLACE FUNCTION collection_geom(content jsonb)
151+
RETURNS geometry AS $$
152+
WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box)
153+
SELECT
154+
st_makeenvelope(
155+
(box->>0)::float,
156+
(box->>1)::float,
157+
(box->>2)::float,
158+
(box->>3)::float,
159+
4326
160+
)
161+
FROM box;
162+
$$ LANGUAGE SQL IMMUTABLE STRICT;
163+
164+
CREATE OR REPLACE FUNCTION collection_datetime(content jsonb)
165+
RETURNS timestamptz AS $$
166+
SELECT
167+
CASE
168+
WHEN
169+
(content->'extent'->'temporal'->'interval'->0->>0) IS NULL
170+
THEN '-infinity'::timestamptz
171+
ELSE
172+
(content->'extent'->'temporal'->'interval'->0->>0)::timestamptz
173+
END
174+
;
175+
$$ LANGUAGE SQL IMMUTABLE STRICT;
176+
177+
CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb)
178+
RETURNS timestamptz AS $$
179+
SELECT
180+
CASE
181+
WHEN
182+
(content->'extent'->'temporal'->'interval'->0->>1) IS NULL
183+
THEN 'infinity'::timestamptz
184+
ELSE
185+
(content->'extent'->'temporal'->'interval'->0->>1)::timestamptz
186+
END
187+
;
188+
$$ LANGUAGE SQL IMMUTABLE STRICT;
189+
-- BEGIN migra calculated SQL
190+
set check_function_bodies = off;
191+
192+
CREATE OR REPLACE FUNCTION pgstac.items_staging_triggerfunc()
193+
RETURNS trigger
194+
LANGUAGE plpgsql
195+
AS $function$
196+
DECLARE
197+
p record;
198+
_partitions text[];
199+
part text;
200+
ts timestamptz := clock_timestamp();
201+
nrows int;
202+
BEGIN
203+
RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts;
204+
205+
FOR part IN WITH t AS (
206+
SELECT
207+
n.content->>'collection' as collection,
208+
stac_daterange(n.content->'properties') as dtr,
209+
partition_trunc
210+
FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id)
211+
), p AS (
212+
SELECT
213+
collection,
214+
COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d,
215+
tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange,
216+
tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange
217+
FROM t
218+
GROUP BY 1,2
219+
) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP
220+
RAISE NOTICE 'Partition %', part;
221+
END LOOP;
222+
223+
RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts;
224+
DROP TABLE IF EXISTS tmpdata;
225+
CREATE TEMP TABLE tmpdata ON COMMIT DROP AS
226+
SELECT
227+
(content_dehydrate(content)).*
228+
FROM newdata;
229+
GET DIAGNOSTICS nrows = ROW_COUNT;
230+
RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts;
231+
232+
RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts;
233+
IF TG_TABLE_NAME = 'items_staging' THEN
234+
INSERT INTO items
235+
SELECT * FROM tmpdata;
236+
GET DIAGNOSTICS nrows = ROW_COUNT;
237+
RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts;
238+
ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN
239+
INSERT INTO items
240+
SELECT * FROM tmpdata
241+
ON CONFLICT DO NOTHING;
242+
GET DIAGNOSTICS nrows = ROW_COUNT;
243+
RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts;
244+
ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN
245+
DELETE FROM items i USING tmpdata s
246+
WHERE
247+
i.id = s.id
248+
AND i.collection = s.collection
249+
AND i IS DISTINCT FROM s
250+
;
251+
GET DIAGNOSTICS nrows = ROW_COUNT;
252+
RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts;
253+
INSERT INTO items AS t
254+
SELECT * FROM tmpdata
255+
ON CONFLICT DO NOTHING;
256+
GET DIAGNOSTICS nrows = ROW_COUNT;
257+
RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts;
258+
END IF;
259+
260+
RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts;
261+
DELETE FROM items_staging;
262+
RAISE NOTICE 'Done. %', clock_timestamp() - ts;
263+
264+
RETURN NULL;
265+
266+
END;
267+
$function$
268+
;
269+
270+
271+
-- END migra calculated SQL
272+
DO $$
273+
BEGIN
274+
INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES
275+
('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null);
276+
EXCEPTION WHEN unique_violation THEN
277+
RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE;
278+
END
279+
$$;
280+
281+
DO $$
282+
BEGIN
283+
INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES
284+
('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null);
285+
EXCEPTION WHEN unique_violation THEN
286+
RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE;
287+
END
288+
$$;
289+
290+
DO $$
291+
BEGIN
292+
INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES
293+
('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null);
294+
EXCEPTION WHEN unique_violation THEN
295+
RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE;
296+
END
297+
$$;
298+
299+
DELETE FROM queryables a USING queryables b
300+
WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id;
301+
302+
303+
INSERT INTO pgstac_settings (name, value) VALUES
304+
('context', 'off'),
305+
('context_estimated_count', '100000'),
306+
('context_estimated_cost', '100000'),
307+
('context_stats_ttl', '1 day'),
308+
('default_filter_lang', 'cql2-json'),
309+
('additional_properties', 'true'),
310+
('use_queue', 'false'),
311+
('queue_timeout', '10 minutes'),
312+
('update_collection_extent', 'false'),
313+
('format_cache', 'false'),
314+
('readonly', 'false')
315+
ON CONFLICT DO NOTHING
316+
;
317+
318+
ALTER FUNCTION to_text COST 5000;
319+
ALTER FUNCTION to_float COST 5000;
320+
ALTER FUNCTION to_int COST 5000;
321+
ALTER FUNCTION to_tstz COST 5000;
322+
ALTER FUNCTION to_text_array COST 5000;
323+
324+
ALTER FUNCTION update_partition_stats SECURITY DEFINER;
325+
ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER;
326+
ALTER FUNCTION drop_table_constraints SECURITY DEFINER;
327+
ALTER FUNCTION create_table_constraints SECURITY DEFINER;
328+
ALTER FUNCTION check_partition SECURITY DEFINER;
329+
ALTER FUNCTION repartition SECURITY DEFINER;
330+
ALTER FUNCTION where_stats SECURITY DEFINER;
331+
ALTER FUNCTION search_query SECURITY DEFINER;
332+
ALTER FUNCTION format_item SECURITY DEFINER;
333+
ALTER FUNCTION maintain_index SECURITY DEFINER;
334+
335+
GRANT USAGE ON SCHEMA pgstac to pgstac_read;
336+
GRANT ALL ON SCHEMA pgstac to pgstac_ingest;
337+
GRANT ALL ON SCHEMA pgstac to pgstac_admin;
338+
339+
-- pgstac_read role limited to using function apis
340+
GRANT EXECUTE ON FUNCTION search TO pgstac_read;
341+
GRANT EXECUTE ON FUNCTION search_query TO pgstac_read;
342+
GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read;
343+
GRANT EXECUTE ON FUNCTION get_item TO pgstac_read;
344+
GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read;
345+
346+
347+
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest;
348+
GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest;
349+
GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest;
350+
351+
REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public;
352+
GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin;
353+
354+
REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public;
355+
GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin;
356+
357+
RESET ROLE;
358+
359+
SET ROLE pgstac_ingest;
360+
SELECT update_partition_stats_q(partition) FROM partitions_view;
361+
SELECT set_version('unreleased');

0 commit comments

Comments
 (0)