Этап 3. Слой Core — обогащённые данные. Пет проект: DWH. РАБОТА НАД ОШИБКАМИ - сделал insert/update в core, добавил индексы.

mnoskov/greenplum/Пет проект: DWH/Step_3_core_layer.sql

@@ -43,6 +43,7 @@ CREATE TABLE core.fact_lesson_views (
 	course_id bigint NOT NULL REFERENCES core.dim_course(course_id),
 	viewed_at timestamptz NOT NULL
 ) DISTRIBUTED BY (user_id);
+CREATE INDEX idx_fact_lesson_views_viewed_at ON core.fact_lesson_views(viewed_at);
 
 --DROP TABLE IF EXISTS core.enrollments;
 CREATE TABLE core.enrollments (
@@ -63,8 +64,8 @@ SELECT
 	raw_u.registration_date
 FROM
 	raw.users raw_u
-WHERE NOT EXISTS
+WHERE
-	(SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id)
+	NOT EXISTS (SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id)
 	AND raw_u.email IS NOT NULL AND raw_u.registration_date IS NOT NULL;
 
 UPDATE core.dim_user core_u
@@ -93,9 +94,9 @@ SELECT
 	raw_c.created_at
 FROM
 	raw.courses raw_c
-WHERE NOT EXISTS
+WHERE
-	(SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id)
+	NOT EXISTS (SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id)
-	AND raw_c.title IS NOT NULL AND raw_c;
+	AND raw_c.title IS NOT NULL AND raw_c.created_at IS NOT NULL;
 
 UPDATE core.dim_course core_c
 SET
@@ -121,8 +122,8 @@ SELECT
 	raw_l.course_id
 FROM
 	raw.lessons raw_l
-WHERE NOT EXISTS
+WHERE
-	(SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id)
+	NOT EXISTS (SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id)
 	AND raw_l.title IS NOT NULL AND raw_l.course_id IS NOT NULL;
 
 UPDATE core.dim_lesson core_l
@@ -140,7 +141,7 @@ WHERE
 		core_l.course_id != raw_l.course_id
 	);
 
--- загуглил как заполнять:
+-- загуглил как заполнять dim_date:
 INSERT INTO core.dim_date (date, year, month, day, weekday)
 SELECT
 	d::date,
@@ -150,23 +151,32 @@ SELECT
 	to_char(d, 'day')
 FROM
 	generate_series('2023-01-01'::date, '2030-12-31'::date, INTERVAL '1 day') AS d;
--- получается dim_date загружается один раз, и должно храниться где-то отдельно?
 
 INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
 SELECT
-	lv.user_id::int,
+	raw_lv.user_id,
-	lv.lesson_id::int,
+	raw_lv.lesson_id,
-	l.course_id::int,
+	raw_l.course_id,
-	lv.viewed_at::timestamp
+	raw_lv.viewed_at
 FROM
-	raw.lesson_views lv
+	raw.lesson_views raw_lv
 JOIN
-	raw.lessons l ON lv.lesson_id = l.id;
+	raw.lessons raw_l ON raw_lv.lesson_id = raw_l.id
+WHERE
+	raw_lv.viewed_at > (SELECT max(viewed_at) FROM core.fact_lesson_views) -- но при первой загрузке это не сработало, тк max(vied_at) = NULL, как это исправить - я не знаю
+	AND raw_lv.user_id IS NOT NULL AND raw_lv.lesson_id IS NOT NULL
+	AND raw_l.course_id IS NOT NULL AND raw_lv.viewed_at IS NOT NULL;
 
 INSERT INTO core.enrollments (user_id, course_id, enrolled_at)
 SELECT
-	user_id::int,
+	raw_e.user_id,
-	course_id::int,
+	raw_e.course_id,
-	enrolled_at::timestamp
+	raw_e.enrolled_at
 FROM
-	raw.enrollments;
+	raw.enrollments raw_e
+WHERE
+	NOT EXISTS (SELECT 1 FROM core.enrollments core_e WHERE 
+	core_e.user_id = raw_e.user_id
+	AND core_e.course_id = raw_e.course_id
+	AND core_e.enrolled_at = raw_e.enrolled_at)
+	AND raw_e.user_id IS NOT NULL AND raw_e.course_id IS NOT NULL AND raw_e.enrolled_at IS NOT NULL;