Merge pull request 'OS-253' (#24) from OS-253 into main

Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/24

.gitignore

@@ -0,0 +1,2 @@
+itikhomirov/.DS_Store
+.DS_Store

itikhomirov/greenplum/pet_gp_infra.sql

@@ -0,0 +1,241 @@
+-- Этап 1. Инициализация проекта
+-- Задание 1: Создай схемы
+
+CREATE SCHEMA raw;
+CREATE SCHEMA core;
+CREATE SCHEMA dm;
+
+-- Задание 2: Создай raw-таблицы
+
+CREATE TABLE raw.users (
+	id int,
+	name varchar(15),
+	age int,
+	email varchar(30),
+	registration_date DATE
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.courses (
+	id int,
+	title text,
+	category varchar(30),
+	created_at DATE
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.lessons (
+	id int,
+	course_id int,
+	title text,
+	duration_min int
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.enrollments (
+	id int,
+	user_id int,
+	course_id int,
+	enrolled_at DATE
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.lesson_views (
+	id int,
+	user_id int,
+	lesson_id int,
+	viewed_at timestamp
+)
+DISTRIBUTED BY (id);
+
+-- Этап 2. Загрузка данных (эмуляция Airflow)
+-- Задание 3: Выгрузить данные из PostgreSQL и Загрузить CSV в Greenplum
+
+
+-- Экспортировал данные из PostgreSQL в CSV-файлы и загрузил их в Greenplum
+
+-- Этап 3. Слой Core — обогащённые данные
+-- Задание 4: Измерения (dimension tables)
+
+-- Измерение пользователей
+CREATE TABLE core.dim_user (
+    user_id INT PRIMARY KEY,
+    name TEXT,
+    age INT,
+    email TEXT,
+    registration_date DATE
+);
+
+-- Измерение курсов
+CREATE TABLE core.dim_course (
+    course_id INT PRIMARY KEY,
+    title TEXT,
+    category TEXT
+);
+
+-- Измерение уроков
+CREATE TABLE core.dim_lesson (
+    lesson_id INT PRIMARY KEY,
+    title TEXT,
+    duration_min INT,
+    course_id INT REFERENCES core.dim_course(course_id)
+);
+
+-- Опционально: измерение времени
+CREATE TABLE core.dim_date (
+    date DATE PRIMARY KEY,
+    year INT,
+    month INT,
+    day INT,
+    weekday TEXT
+);
+
+-- Задание 5: Факт (fact table)
+
+CREATE TABLE core.fact_lesson_views (
+    user_id INT REFERENCES core.dim_user(user_id),
+    lesson_id INT REFERENCES core.dim_lesson(lesson_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    viewed_at TIMESTAMP
+);
+
+CREATE TABLE core.enrollments (
+    user_id INT REFERENCES core.dim_user(user_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    enrolled_at TIMESTAMP
+);
+
+-- Задание 6: ETL переброса данных из raw в core
+
+INSERT INTO core.dim_user (user_id, name, age, email, registration_date) -- ok
+SELECT DISTINCT
+    id::INT,
+    name::TEXT,
+    age::INT,
+    email::TEXT,
+    registration_date::DATE
+FROM raw.users;
+
+INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) -- ok
+SELECT DISTINCT
+	id::INT,
+	title::TEXT,
+	duration_min::INT,
+	course_id::INT
+FROM raw.lessons;
+
+INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) --заполнить из нескольких таблиц с помощью join
+SELECT DISTINCT
+	lv.user_id::INT,
+	lv.lesson_id::INT,
+	le.course_id::INT,
+	lv.viewed_at::TIMESTAMP
+FROM raw.lesson_views AS lv
+JOIN raw.lessons AS le ON lv.lesson_id = le.id;
+
+INSERT INTO core.enrollments (user_id, course_id, enrolled_at) -- ok
+SELECT DISTINCT 
+	user_id::INT,
+	course_id::INT,
+	enrolled_at::DATE
+FROM raw.enrollments;
+
+INSERT INTO core.dim_course (course_id, title, category) -- ok
+SELECT DISTINCT 
+	id::INT,
+	title::TEXT,
+	category::VARCHAR
+FROM raw.courses;
+
+INSERT INTO core.dim_date (date, year, month, day, weekday)
+SELECT DISTINCT
+	d::DATE,
+	EXTRACT(YEAR FROM d)::INT,
+	EXTRACT(MONTH FROM d)::INT,
+	EXTRACT(DAY FROM d)::INT,
+	TO_CHAR(d, 'Day')::TEXT
+FROM ( 
+	SELECT registration_date AS d FROM raw.users
+	UNION
+	SELECT enrolled_at FROM raw.enrollments
+	UNION
+	SELECT viewed_at FROM raw.lesson_views
+) all_dates
+WHERE d IS NOT NULL;
+
+-- Надо сделать, чтобы вместо номера месяца выводилось название месяца
+
+ALTER TABLE core.dim_date
+ALTER COLUMN month TYPE text;
+
+UPDATE core.dim_date 
+SET month = TO_CHAR(date, 'Month');
+
+-- Этап 4: Подготовка витрин в Greenplum
+-- Задание 7: Витрина dm.lesson_popularity_summary
+
+CREATE MATERIALIZED VIEW dm.lesson_popularity_summary AS
+SELECT 
+	dl.lesson_id,
+	dl.title AS lesson_title,
+	dl.course_id,
+	dc.title AS course_title,
+	COUNT(flv.user_id) AS total_views,
+	COUNT(DISTINCT flv.user_id) AS unique_users,
+	MIN(flv.viewed_at) AS first_view,
+	MAX(flv.viewed_at) AS last_view
+FROM core.fact_lesson_views AS flv
+JOIN core.dim_lesson AS dl ON flv.lesson_id = dl.lesson_id
+JOIN core.dim_course AS dc ON dl.course_id = dc.course_id
+GROUP BY dl.lesson_id, dl.title, dl.course_id, dc.title
+ORDER BY dl.lesson_id ASC;
+
+-- Задание 8: Витрина dm.inactive_users_summary
+
+CREATE MATERIALIZED VIEW dm.inactive_users_summary AS
+SELECT
+	du.user_id,
+	du.name AS user_name,
+	du.email,
+	du.age,
+	COUNT(e.course_id) AS enrollments_count,
+	du.registration_date
+FROM core.dim_user AS du
+LEFT JOIN core.enrollments AS e ON du.user_id = e.user_id
+LEFT JOIN core.fact_lesson_views AS flv ON du.user_id = flv.user_id
+WHERE flv.user_id IS NULL
+GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date;
+
+-- Задание 9: Витрина dm.course_completion_rate
+
+CREATE MATERIALIZED VIEW dm.course_completion_rate AS
+SELECT 
+	du.user_id,
+	du.name AS user_name,
+	dc.course_id,
+	dc.title AS course_title,
+	COUNT(DISTINCT dl.lesson_id) AS lesson_in_course,
+	COUNT(DISTINCT flv.lesson_id) AS lessons_viewed,
+	ROUND(
+		COALESCE(
+			NULLIF(COUNT(DISTINCT flv.lesson_id)::NUMERIC,0) / NULLIF(COUNT(DISTINCT dl.lesson_id), 0),
+			0
+		) ,2
+	) AS completion_rate
+FROM core.dim_user AS du
+JOIN core.enrollments AS e ON du.user_id = e.user_id
+JOIN core.dim_course AS dc ON e.course_id= dc.course_id
+JOIN core.dim_lesson AS dl ON dc.course_id = dl.course_id
+LEFT JOIN core.fact_lesson_views AS flv ON du.user_id = flv.user_id AND dc.course_id = flv.course_id
+GROUP BY du.user_id, user_name, dc.course_id, course_title
+ORDER BY du.user_id ASC;
+
+-- 🧪 Этап 5. Проверка
+-- Задание 10: Проверь витрины
+
+SELECT * FROM dm.lesson_popularity_summary LIMIT 10;
+SELECT * FROM dm.inactive_users_summary LIMIT 10;
+SELECT * FROM dm.course_completion_rate ORDER BY user_id ASC LIMIT 10;
+
+
+