Этап 3. Слой Core — обогащённые данные. Пет проект: DWH. РАБОТА НАД ОШИБКАМИ - изменил логику загрузи и обновления измерений.

mnoskov/greenplum/Пет проект: DWH/Step_3_core_layer.sql

@@ -23,6 +23,7 @@ CREATE TABLE core.dim_lesson (
 	duration_min smallint,
 	course_id bigint NOT NULL REFERENCES core.dim_course(course_id)
 ) DISTRIBUTED BY (lesson_id);
+CREATE INDEX idx_dim_lesson_course_id ON core.dim_lesson(course_id);
 
 --DROP TABLE IF EXISTS core.dim_date;
 CREATE TABLE core.dim_date (
@@ -52,34 +53,92 @@ CREATE TABLE core.enrollments (
 
 
 --6
+-- dim_user
 INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
-SELECT DISTINCT
-	id::int AS user_id,
-	name::text,
-	age::int,
-	email::text,
-	registration_date::date
-FROM raw.users
-WHERE id IS NOT NULL AND email IS NOT NULL;
-
-INSERT INTO core.dim_course (course_id, title, category)
-SELECT DISTINCT
-	id::int AS course_id,
-	title::text,
-	category::text
+SELECT
+	raw_u.id,
+	raw_u.name,
+	raw_u.age,
+	raw_u.email,
+	raw_u.registration_date
 FROM
-	raw.courses
-WHERE id IS NOT NULL AND title IS NOT NULL;
+	raw.users raw_u
+WHERE NOT EXISTS
+	(SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id)
+	AND raw_u.email IS NOT NULL AND raw_u.registration_date IS NOT NULL;
 
+UPDATE core.dim_user core_u
+SET
+	name = raw_u.name,
+	age = raw_u.age,
+	email = raw_u.email,
+	registration_date = raw_u.registration_date
+FROM
+	raw.users raw_u
+WHERE
+	core_u.user_id = raw_u.id
+	AND (
+		core_u.name != raw_u.name OR
+		core_u.age != raw_u.age OR
+		core_u.email != raw_u.email OR
+		core_u.registration_date != raw_u.registration_date
+	);
+
+-- dim_course
+INSERT INTO core.dim_course (course_id, title, category, created_at)
+SELECT
+	raw_c.id,
+	raw_c.title,
+	raw_c.category,
+	raw_c.created_at
+FROM
+	raw.courses raw_c
+WHERE NOT EXISTS
+	(SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id)
+	AND raw_c.title IS NOT NULL AND raw_c;
+
+UPDATE core.dim_course core_c
+SET
+	title = raw_c.title,
+	category = raw_c.category,
+	created_at = raw_c.created_at
+FROM
+	raw.courses raw_c
+WHERE
+	core_c.course_id = raw_c.id
+	AND (
+		core_c.title != raw_c.title OR
+		core_c.category != raw_c.category OR
+		core_c.created_at != raw_c.created_at
+	);
+
+-- dim_lesson
 INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
-SELECT DISTINCT
-	id::int AS lesson_id,
-	title::text,
-	duration_min::int,
-	course_id::int
+SELECT
+	raw_l.id,
+	raw_l.title,
+	raw_l.duration_min,
+	raw_l.course_id
 FROM
-	raw.lessons
-WHERE id IS NOT NULL;
+	raw.lessons raw_l
+WHERE NOT EXISTS
+	(SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id)
+	AND raw_l.title IS NOT NULL AND raw_l.course_id IS NOT NULL;
+
+UPDATE core.dim_lesson core_l
+SET
+	title = raw_l.title,
+	duration_min = raw_l.duration_min,
+	course_id = raw_l.course_id -- решил оставить, тк чисто теоретические же урок может начать принадлежать другому курсу
+FROM
+	raw.lessons raw_l
+WHERE
+	core_l.lesson_id = raw_l.id
+	AND (
+		core_l.title != raw_l.title OR
+		core_l.duration_min != raw_l.duration_min OR
+		core_l.course_id != raw_l.course_id
+	);
 
 -- загуглил как заполнять:
 INSERT INTO core.dim_date (date, year, month, day, weekday)
@@ -90,7 +149,8 @@ SELECT
 	EXTRACT(DAY FROM d)::int,
 	to_char(d, 'day')
 FROM
-	generate_series('2023-01-01'::date, '2025-12-31'::date, INTERVAL '1 day') AS d;
+	generate_series('2023-01-01'::date, '2030-12-31'::date, INTERVAL '1 day') AS d;
+-- получается dim_date загружается один раз, и должно храниться где-то отдельно?
 
 INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
 SELECT