Skip to content

Commit a8d581f

Browse files
EvolveArtEvolveArt
committed
fix: migrations bugs
1 parent 72195af commit a8d581f

File tree

7 files changed

+52
-58
lines changed

7 files changed

+52
-58
lines changed

‎compose.dev.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
services:
22
onchain-db:
3-
image: timescale/timescaledb-ha:pg14-latest
3+
image: timescale/timescaledb-ha:pg17.4-ts2.18.2
44
restart: always
55
user: postgres
66
networks:
@@ -37,7 +37,7 @@ services:
3737
retries: 5
3838

3939
offchain-db:
40-
image: timescale/timescaledb-ha:pg14-latest
40+
image: timescale/timescaledb-ha:pg17.4-ts2.18.2
4141
restart: always
4242
user: postgres
4343
networks:

‎pragma-entities/migrations/2023-10-11-223513_create_entries/up.sql

+3-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ CREATE TABLE entries (
99
price NUMERIC NOT NULL,
1010
timestamp TIMESTAMPTZ NOT NULL,
1111
publisher TEXT NOT NULL,
12-
publisher_signature TEXT NOT NULL,
12+
publisher_signature TEXT NULLABLE,
1313
source VARCHAR NOT NULL,
1414
PRIMARY KEY (id, timestamp)
1515
);
@@ -28,9 +28,9 @@ CREATE TABLE future_entries (
2828
pair_id VARCHAR NOT NULL,
2929
price NUMERIC NOT NULL,
3030
timestamp TIMESTAMPTZ NOT NULL,
31-
expiration_timestamp TIMESTAMPTZ, -- can be NULL for perp contracts
31+
expiration_timestamp TIMESTAMPTZ NULLABLE, -- can be NULL for perp contracts
3232
publisher TEXT NOT NULL,
33-
publisher_signature TEXT NOT NULL,
33+
publisher_signature TEXT NULLABLE,
3434
source VARCHAR NOT NULL,
3535
PRIMARY KEY (id, timestamp)
3636
);

‎pragma-entities/migrations/2024-01-11-123510_add_continuous_aggregates/up.sql

+8-6
Original file line numberDiff line numberDiff line change
@@ -7,28 +7,29 @@ CREATE OR REPLACE FUNCTION create_continuous_aggregate(
77
RETURNS void AS $$
88
BEGIN
99
EXECUTE format('
10-
CREATE MATERIALIZED VIEW %s
10+
CREATE MATERIALIZED VIEW %I
1111
WITH (timescaledb.continuous, timescaledb.materialized_only = false)
1212
AS SELECT
1313
pair_id,
14-
time_bucket($1::interval, timestamp) as bucket,
14+
time_bucket(%L, timestamp) as bucket,
1515
(percentile_cont(0.5) WITHIN GROUP (ORDER BY price))::numeric(1000,0) AS median_price,
1616
COUNT(DISTINCT source) as num_sources
1717
FROM %I
1818
GROUP BY bucket, pair_id
1919
WITH NO DATA;', p_name, p_interval, p_table_name);
2020

2121
EXECUTE format('
22-
SELECT add_continuous_aggregate_policy(''%s'',
23-
start_offset => $1,
24-
end_offset => $2,
25-
schedule_interval => $3);', p_name, p_start_offset, p_interval, p_interval);
22+
SELECT add_continuous_aggregate_policy(%L,
23+
start_offset => %L,
24+
end_offset => %L,
25+
schedule_interval => %L);', p_name, p_start_offset, '0'::interval, p_interval);
2626
END;
2727
$$ LANGUAGE plpgsql;
2828

2929
-- Spot entries aggregates
3030
SELECT create_continuous_aggregate('price_100_ms_agg', '100 milliseconds'::interval, '300 milliseconds'::interval, 'entries');
3131
SELECT create_continuous_aggregate('price_1_s_agg', '1 second'::interval, '3 seconds'::interval, 'entries');
32+
SELECT create_continuous_aggregate('price_10_s_agg', '10 seconds'::interval, '30 seconds'::interval, 'entries');
3233
SELECT create_continuous_aggregate('price_5_s_agg', '5 seconds'::interval, '15 seconds'::interval, 'entries');
3334
SELECT create_continuous_aggregate('price_1_min_agg', '1 minute'::interval, '3 minutes'::interval, 'entries');
3435
SELECT create_continuous_aggregate('price_15_min_agg', '15 minutes'::interval, '45 minutes'::interval, 'entries');
@@ -40,6 +41,7 @@ SELECT create_continuous_aggregate('price_1_week_agg', '1 week'::interval, '3 we
4041
-- Future entries aggregates
4142
SELECT create_continuous_aggregate('price_100_ms_agg_future', '100 milliseconds'::interval, '300 milliseconds'::interval, 'future_entries');
4243
SELECT create_continuous_aggregate('price_1_s_agg_future', '1 second'::interval, '3 seconds'::interval, 'future_entries');
44+
SELECT create_continuous_aggregate('price_10_s_agg_future', '10 seconds'::interval, '30 seconds'::interval, 'future_entries');
4345
SELECT create_continuous_aggregate('price_5_s_agg_future', '5 seconds'::interval, '15 seconds'::interval, 'future_entries');
4446
SELECT create_continuous_aggregate('price_1_min_agg_future', '1 minute'::interval, '3 minutes'::interval, 'future_entries');
4547
SELECT create_continuous_aggregate('price_15_min_agg_future', '15 minutes'::interval, '45 minutes'::interval, 'future_entries');

‎pragma-entities/migrations/2024-01-12-135355_add_candlestick_views/up.sql

+11-11
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,24 @@ CREATE OR REPLACE FUNCTION create_candlestick_view(
77
RETURNS void AS $$
88
BEGIN
99
EXECUTE format('
10-
CREATE MATERIALIZED VIEW %s
10+
CREATE MATERIALIZED VIEW %I
1111
WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS
1212
SELECT
13-
time_bucket($1::interval, timestamp) AS bucket,
13+
time_bucket(%L, bucket) AS ohlc_bucket,
1414
pair_id,
15-
FIRST(price, timestamp)::numeric AS "open",
16-
MAX(price)::numeric AS high,
17-
MIN(price)::numeric AS low,
18-
LAST(price, timestamp)::numeric AS "close"
15+
FIRST(median_price, bucket)::numeric AS "open",
16+
MAX(median_price)::numeric AS high,
17+
MIN(median_price)::numeric AS low,
18+
LAST(median_price, bucket)::numeric AS "close"
1919
FROM %I
20-
GROUP BY bucket, pair_id
20+
GROUP BY ohlc_bucket, pair_id
2121
WITH NO DATA;', p_name, p_interval, p_table_name);
2222

2323
EXECUTE format('
24-
SELECT add_continuous_aggregate_policy(''%s'',
25-
start_offset => $1,
26-
end_offset => $2,
27-
schedule_interval => $3);', p_name, p_start_offset, p_interval, p_interval);
24+
SELECT add_continuous_aggregate_policy(%L,
25+
start_offset => %L,
26+
end_offset => %L,
27+
schedule_interval => %L);', p_name, p_start_offset, '0'::interval, p_interval);
2828
END;
2929
$$ LANGUAGE plpgsql;
3030

‎pragma-entities/migrations/2024-01-19-113453_add_twap_aggregates/up.sql

+6-6
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,22 @@ CREATE OR REPLACE FUNCTION create_twap_aggregate(
77
RETURNS void AS $$
88
BEGIN
99
EXECUTE format('
10-
CREATE MATERIALIZED VIEW %s
10+
CREATE MATERIALIZED VIEW %I
1111
WITH (timescaledb.continuous, timescaledb.materialized_only = false)
1212
AS SELECT
1313
pair_id,
14-
time_bucket($1::interval, timestamp) as bucket,
14+
time_bucket(%L, timestamp) as bucket,
1515
average(time_weight(''Linear'', timestamp, price))::numeric as price_twap,
1616
COUNT(DISTINCT source) as num_sources
1717
FROM %I
1818
GROUP BY bucket, pair_id
1919
WITH NO DATA;', p_name, p_interval, p_table_name);
2020

2121
EXECUTE format('
22-
SELECT add_continuous_aggregate_policy(''%s'',
23-
start_offset => $1,
24-
end_offset => $2,
25-
schedule_interval => $3);', p_name, p_start_offset, p_interval, p_interval);
22+
SELECT add_continuous_aggregate_policy(%L,
23+
start_offset => %L,
24+
end_offset => %L,
25+
schedule_interval => %L);', p_name, p_start_offset, '0'::interval, p_interval);
2626
END;
2727
$$ LANGUAGE plpgsql;
2828

‎pragma-entities/migrations/2025-03-15-043705_add-compression/down.sql

+11-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,14 @@ BEGIN
3737
'price_1_week_agg', 'price_1_week_agg_future'
3838
]))
3939
LOOP
40-
-- Remove compression policy
41-
CALL remove_compression_policy(view_name, if_exists => true);
40+
BEGIN
41+
-- Remove compression policy if it exists
42+
CALL remove_columnstore_policy(view_name);
43+
EXCEPTION WHEN OTHERS THEN
44+
-- Skip if policy doesn't exist
45+
RAISE NOTICE 'No compression policy found for %', view_name;
46+
END;
47+
4248
-- Disable columnstore
4349
EXECUTE format('ALTER MATERIALIZED VIEW %I SET (timescaledb.enable_columnstore = false)', view_name);
4450
END LOOP;
@@ -52,6 +58,9 @@ SELECT remove_compression_from_continuous_aggregates();
5258
DROP FUNCTION IF EXISTS add_compression_to_continuous_aggregates;
5359
DROP FUNCTION IF EXISTS remove_compression_from_continuous_aggregates;
5460

61+
CALL remove_columnstore_policy('entries');
62+
CALL remove_columnstore_policy('future_entries');
63+
5564
-- Remove compression from base tables
5665
ALTER TABLE entries SET (timescaledb.enable_columnstore = false);
5766
ALTER TABLE future_entries SET (timescaledb.enable_columnstore = false);

‎pragma-entities/migrations/2025-03-15-043705_add-compression/up.sql

+11-28
Original file line numberDiff line numberDiff line change
@@ -22,64 +22,47 @@ DECLARE
2222
compress_after interval;
2323
BEGIN
2424
FOR view_name IN
25-
SELECT format('%s', unnest(ARRAY[
26-
-- Sub-minute aggregates
25+
SELECT unnest(ARRAY[
2726
'price_100_ms_agg', 'price_1_s_agg', 'price_5_s_agg',
2827
'price_100_ms_agg_future', 'price_1_s_agg_future', 'price_5_s_agg_future',
2928
'candle_10_s', 'candle_10_s_future',
30-
31-
-- 1-15min aggregates
3229
'price_1_min_agg', 'price_15_min_agg',
3330
'price_1_min_agg_future', 'price_15_min_agg_future',
3431
'candle_1_min', 'candle_5_min', 'candle_15_min',
3532
'candle_1_min_future', 'candle_5_min_future', 'candle_15_min_future',
3633
'twap_1_min_agg', 'twap_5_min_agg', 'twap_15_min_agg',
3734
'twap_1_min_agg_future', 'twap_5_min_agg_future', 'twap_15_min_agg_future',
38-
39-
-- 1-2h aggregates
4035
'price_1_h_agg', 'price_2_h_agg',
4136
'price_1_h_agg_future', 'price_2_h_agg_future',
4237
'candle_1_h', 'candle_1_h_future',
4338
'twap_1_h_agg', 'twap_2_h_agg',
4439
'twap_1_h_agg_future', 'twap_2_h_agg_future',
45-
46-
-- Daily aggregates
4740
'price_1_day_agg', 'price_1_day_agg_future',
4841
'candle_1_day', 'candle_1_day_future',
4942
'twap_1_day_agg', 'twap_1_day_agg_future',
50-
51-
-- Weekly aggregates
5243
'price_1_week_agg', 'price_1_week_agg_future'
53-
]))
44+
])
5445
LOOP
55-
-- Check if view exists
56-
IF NOT EXISTS (SELECT 1 FROM pg_matviews WHERE matviewname = view_name) THEN
57-
RAISE NOTICE 'View % does not exist, skipping...', view_name;
58-
CONTINUE;
59-
END IF;
60-
6146
-- Set compression interval based on view name pattern
6247
compress_after :=
6348
CASE
64-
WHEN view_name LIKE '%100_ms%' OR view_name LIKE '%_s_%' THEN INTERVAL '1 hour'
49+
WHEN view_name LIKE '%100_ms%' OR view_name LIKE '%s%' THEN INTERVAL '1 hour'
6550
WHEN view_name LIKE '%min%' THEN INTERVAL '6 hours'
66-
WHEN view_name LIKE '%_h_%' OR view_name LIKE '%_2_h%' THEN INTERVAL '1 day'
51+
WHEN view_name LIKE '%h%' OR view_name LIKE '%2_h%' THEN INTERVAL '1 day'
6752
WHEN view_name LIKE '%day%' THEN INTERVAL '7 days'
6853
WHEN view_name LIKE '%week%' THEN INTERVAL '30 days'
6954
END;
7055

71-
BEGIN
72-
-- Enable columnstore and set segmentby for each view
73-
EXECUTE format('ALTER MATERIALIZED VIEW %I SET (timescaledb.enable_columnstore = true, timescaledb.segmentby = ''pair_id'')', view_name);
74-
-- Add compression policy
75-
EXECUTE format('CALL add_columnstore_policy(%L, after => $1, if_not_exists => true)', view_name) USING compress_after;
76-
EXCEPTION
77-
WHEN OTHERS THEN
78-
RAISE WARNING 'Failed to add compression policy for view %: %', view_name, SQLERRM;
79-
END;
56+
-- Enable columnstore and set segmentby for each view
57+
EXECUTE format('ALTER MATERIALIZED VIEW %I SET (timescaledb.enable_columnstore = true, timescaledb.segmentby = ''pair_id'')', view_name);
58+
59+
-- Add compression policy
60+
EXECUTE format('CALL add_columnstore_policy(%I, compress_after => $1)', view_name) USING compress_after;
61+
8062
END LOOP;
8163
END;
8264
$$ LANGUAGE plpgsql;
8365

66+
8467
-- Add compression policies to all continuous aggregates
8568
SELECT add_compression_to_continuous_aggregates();

0 commit comments

Comments
 (0)