dmitry_medvedev/homework
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Этап 3. Слой Core — обогащённые данные. Пет проект: DWH. РАБОТА НАД ОШИБКАМИ - сделал insert/update в core, добавил индексы.

Browse Source
This commit is contained in:
mnoskov
2026-04-25 18:11:21 +03:00
parent 95456999b3
commit add8d776e5
1 changed files with 29 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mnoskov/greenplum/Пет проект: DWH/Step_3_core_layer.sql
+29 -19
View File
@@ -43,6 +43,7 @@ CREATE TABLE core.fact_lesson_views (
course_id bigint NOT NULL REFERENCES core.dim_course(course_id),
viewed_at timestamptz NOT NULL
) DISTRIBUTED BY (user_id);
CREATE INDEX idx_fact_lesson_views_viewed_at ON core.fact_lesson_views(viewed_at);
--DROP TABLE IF EXISTS core.enrollments;
CREATE TABLE core.enrollments (
@@ -63,8 +64,8 @@ SELECT
raw_u.registration_date
FROM
raw.users raw_u
WHERE NOT EXISTS
(SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id)
WHERE
NOT EXISTS (SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id)
AND raw_u.email IS NOT NULL AND raw_u.registration_date IS NOT NULL;
UPDATE core.dim_user core_u
@@ -93,9 +94,9 @@ SELECT
raw_c.created_at
FROM
raw.courses raw_c
WHERE NOT EXISTS
(SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id)
AND raw_c.title IS NOT NULL AND raw_c;
WHERE
NOT EXISTS (SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id)
AND raw_c.title IS NOT NULL AND raw_c.created_at IS NOT NULL;
UPDATE core.dim_course core_c
SET
@@ -121,8 +122,8 @@ SELECT
raw_l.course_id
FROM
raw.lessons raw_l
WHERE NOT EXISTS
(SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id)
WHERE
NOT EXISTS (SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id)
AND raw_l.title IS NOT NULL AND raw_l.course_id IS NOT NULL;
UPDATE core.dim_lesson core_l
@@ -140,7 +141,7 @@ WHERE
core_l.course_id != raw_l.course_id
);
-- загуглил как заполнять:
-- загуглил как заполнять dim_date:
INSERT INTO core.dim_date (date, year, month, day, weekday)
SELECT
d::date,
@@ -150,23 +151,32 @@ SELECT
to_char(d, 'day')
FROM
generate_series('2023-01-01'::date, '2030-12-31'::date, INTERVAL '1 day') AS d;
-- получается dim_date загружается один раз, и должно храниться где-то отдельно?
INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
SELECT
lv.user_id::int,
lv.lesson_id::int,
l.course_id::int,
lv.viewed_at::timestamp
raw_lv.user_id,
raw_lv.lesson_id,
raw_l.course_id,
raw_lv.viewed_at
FROM
raw.lesson_views lv
raw.lesson_views raw_lv
JOIN
raw.lessons l ON lv.lesson_id = l.id;
raw.lessons raw_l ON raw_lv.lesson_id = raw_l.id
WHERE
raw_lv.viewed_at > (SELECT max(viewed_at) FROM core.fact_lesson_views) -- но при первой загрузке это не сработало, тк max(vied_at) = NULL, как это исправить - я не знаю
AND raw_lv.user_id IS NOT NULL AND raw_lv.lesson_id IS NOT NULL
AND raw_l.course_id IS NOT NULL AND raw_lv.viewed_at IS NOT NULL;
INSERT INTO core.enrollments (user_id, course_id, enrolled_at)
SELECT
user_id::int,
course_id::int,
enrolled_at::timestamp
raw_e.user_id,
raw_e.course_id,
raw_e.enrolled_at
FROM
raw.enrollments;
raw.enrollments raw_e
WHERE
NOT EXISTS (SELECT 1 FROM core.enrollments core_e WHERE
core_e.user_id = raw_e.user_id
AND core_e.course_id = raw_e.course_id
AND core_e.enrolled_at = raw_e.enrolled_at)
AND raw_e.user_id IS NOT NULL AND raw_e.course_id IS NOT NULL AND raw_e.enrolled_at IS NOT NULL;