diff --git a/mnoskov/greenplum/Пет проект: DWH/Step_3_core_layer.sql b/mnoskov/greenplum/Пет проект: DWH/Step_3_core_layer.sql index 7af360d..c44753d 100644 --- a/mnoskov/greenplum/Пет проект: DWH/Step_3_core_layer.sql +++ b/mnoskov/greenplum/Пет проект: DWH/Step_3_core_layer.sql @@ -43,6 +43,7 @@ CREATE TABLE core.fact_lesson_views ( course_id bigint NOT NULL REFERENCES core.dim_course(course_id), viewed_at timestamptz NOT NULL ) DISTRIBUTED BY (user_id); +CREATE INDEX idx_fact_lesson_views_viewed_at ON core.fact_lesson_views(viewed_at); --DROP TABLE IF EXISTS core.enrollments; CREATE TABLE core.enrollments ( @@ -63,8 +64,8 @@ SELECT raw_u.registration_date FROM raw.users raw_u -WHERE NOT EXISTS - (SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id) +WHERE + NOT EXISTS (SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id) AND raw_u.email IS NOT NULL AND raw_u.registration_date IS NOT NULL; UPDATE core.dim_user core_u @@ -93,9 +94,9 @@ SELECT raw_c.created_at FROM raw.courses raw_c -WHERE NOT EXISTS - (SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id) - AND raw_c.title IS NOT NULL AND raw_c; +WHERE + NOT EXISTS (SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id) + AND raw_c.title IS NOT NULL AND raw_c.created_at IS NOT NULL; UPDATE core.dim_course core_c SET @@ -121,8 +122,8 @@ SELECT raw_l.course_id FROM raw.lessons raw_l -WHERE NOT EXISTS - (SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id) +WHERE + NOT EXISTS (SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id) AND raw_l.title IS NOT NULL AND raw_l.course_id IS NOT NULL; UPDATE core.dim_lesson core_l @@ -140,7 +141,7 @@ WHERE core_l.course_id != raw_l.course_id ); --- загуглил как заполнять: +-- загуглил как заполнять dim_date: INSERT INTO core.dim_date (date, year, month, day, weekday) SELECT d::date, @@ -150,23 +151,32 @@ SELECT to_char(d, 'day') FROM generate_series('2023-01-01'::date, '2030-12-31'::date, INTERVAL '1 day') AS d; --- получается dim_date загружается один раз, и должно храниться где-то отдельно? INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) SELECT - lv.user_id::int, - lv.lesson_id::int, - l.course_id::int, - lv.viewed_at::timestamp + raw_lv.user_id, + raw_lv.lesson_id, + raw_l.course_id, + raw_lv.viewed_at FROM - raw.lesson_views lv + raw.lesson_views raw_lv JOIN - raw.lessons l ON lv.lesson_id = l.id; + raw.lessons raw_l ON raw_lv.lesson_id = raw_l.id +WHERE + raw_lv.viewed_at > (SELECT max(viewed_at) FROM core.fact_lesson_views) -- но при первой загрузке это не сработало, тк max(vied_at) = NULL, как это исправить - я не знаю + AND raw_lv.user_id IS NOT NULL AND raw_lv.lesson_id IS NOT NULL + AND raw_l.course_id IS NOT NULL AND raw_lv.viewed_at IS NOT NULL; INSERT INTO core.enrollments (user_id, course_id, enrolled_at) SELECT - user_id::int, - course_id::int, - enrolled_at::timestamp + raw_e.user_id, + raw_e.course_id, + raw_e.enrolled_at FROM - raw.enrollments; \ No newline at end of file + raw.enrollments raw_e +WHERE + NOT EXISTS (SELECT 1 FROM core.enrollments core_e WHERE + core_e.user_id = raw_e.user_id + AND core_e.course_id = raw_e.course_id + AND core_e.enrolled_at = raw_e.enrolled_at) + AND raw_e.user_id IS NOT NULL AND raw_e.course_id IS NOT NULL AND raw_e.enrolled_at IS NOT NULL; \ No newline at end of file