Merge pull request 'added STEP 1, 3, 4.3' (#32) from OS-233 into main

Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/32 Reviewed-by: aleksey bogatyrev <abogatyrev@example.com>
2025-12-29 16:57:30 +04:00
parent b2fc875eba 4a00777da3
commit a99a7368ae
7 changed files with 257 additions and 0 deletions
@@ -0,0 +1,5 @@
+--- создаем необх. схемы для DWH
+
+CREATE SCHEMA raw;
+CREATE SCHEMA core;
+CREATE SCHEMA dm;
@@ -0,0 +1,44 @@
+--создаем таблицы для raw слоя с DISTRIBUTED REPLICATED, т.к. таблицы довольно маленькие 
+
+
+CREATE TABLE raw.users (
+	id BIGINT PRIMARY KEY, 
+	name VARCHAR(100), 
+	age INT, 
+	email VARCHAR(255) UNIQUE, 
+	registration_date DATE NOT NULL
+)
+DISTRIBUTED REPLICATED;
+
+CREATE TABLE raw.courses(
+	id BIGINT PRIMARY KEY,
+	title TEXT NOT NULL,
+	category VARCHAR(255) CHECK (category IN ('data', 'programming', 'business')),
+	created_at DATE NOT NULL
+	)
+	DISTRIBUTED REPLICATED;
+
+CREATE TABLE raw.lessons(
+	id BIGINT PRIMARY KEY, 
+	course_id INT NOT NULL,
+	title TEXT NOT NULL, 
+	duration_min INT NOT NULL
+	)
+	DISTRIBUTED REPLICATED;
+
+
+CREATE TABLE raw.enrollments(
+	id BIGINT PRIMARY KEY, 
+	user_id INT NOT NULL,
+	course_id INT NOT NULL,
+	enrolled_at DATE NOT NULL
+	)
+	DISTRIBUTED REPLICATED;
+
+CREATE TABLE raw.lesson_views(
+	id BIGINT PRIMARY KEY, 
+	user_id INT NOT NULL,
+	lesson_id INT NOT NULL,
+	viewed_at TIMESTAMP NOT NULL
+	)
+	DISTRIBUTED REPLICATED;
@@ -0,0 +1,95 @@
+--- ИЗМЕРЕНИЕ ПОЛЬЗОВАТЕЛЕЙ, РАСПРЕДЕЛЕНИЕ ПО user_id - на проде табл. может быть большой
+
+CREATE TABLE core.dim_user (
+    user_id INT PRIMARY KEY,
+    name TEXT,
+    age INT,
+    email TEXT,
+    registration_date DATE
+)
+DISTRIBUTED BY (user_id);
+
+INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
+SELECT DISTINCT
+    id::INT,
+    name::TEXT,
+    age::INT,
+    email::TEXT,
+    registration_date::DATE
+FROM raw.users;
+
+
+
+--- ИЗМЕРЕНИЕ КУРСОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник
+
+CREATE TABLE core.dim_course (
+    course_id INT PRIMARY KEY,
+    title TEXT,
+    category TEXT
+)
+DISTRIBUTED REPLICATED;
+
+INSERT INTO core.dim_course (course_id, title, category)
+SELECT DISTINCT
+    id::BIGINT,
+    title::TEXT,
+    category::VARCHAR(255)
+FROM raw.courses;
+
+
+
+--- ИЗМЕРЕНИЕ УРОКОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник
+
+CREATE TABLE core.dim_lesson (
+    lesson_id INT PRIMARY KEY,
+    title TEXT,
+    duration_min INT,
+    course_id INT REFERENCES core.dim_course(course_id)
+)
+DISTRIBUTED REPLICATED;
+
+INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
+SELECT DISTINCT
+    id::BIGINT,
+    title::TEXT,
+    duration_min::INT,
+    course_id::INT
+FROM raw.lessons;
+
+
+---- ФАКТ ПРОСМОТРОВ УРОКОВ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN
+CREATE TABLE core.fact_lesson_views (
+    user_id INT REFERENCES core.dim_user(user_id),
+    lesson_id INT REFERENCES core.dim_lesson(lesson_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    viewed_at TIMESTAMP
+)
+DISTRIBUTED BY (user_id);
+
+INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
+SELECT 
+	lv.user_id::INT,
+	lv.lesson_id::INT,
+	l.course_id::INT,
+	lv.viewed_at::TIMESTAMP
+FROM raw.lesson_views lv
+LEFT JOIN raw.lessons l ON lv.lesson_id = l.id;
+
+
+
+
+---- ФАКТ ЗАПИСИ НА КУРСЫ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN
+CREATE TABLE core.enrollments (
+    user_id INT REFERENCES core.dim_user(user_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    enrolled_at TIMESTAMP
+)
+DISTRIBUTED BY (user_id);
+
+
+INSERT INTO core.enrollments (user_id, course_id, enrolled_at)
+SELECT 
+	user_id::INT,
+	course_id::INT,
+	enrolled_at::DATE
+FROM raw.enrollments
@@ -0,0 +1,42 @@
+/*
+ - ВИТРИНА АНАЛИЗА ПОПУЛЯРНОСТИ КАЖДОГО УРОКА
+ - ВСЕ УЧАСТВУЮЩИЕ ИЗМЕРЕНИЯ DISTRIBUTED REPLICATED
+ - ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ DISTRIBUTED BY (lesson_id, course_id)
+ */
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS dm.lesson_popularity_summary AS
+-- EXPLAIN ANALYZE
+DISTRIBUTED BY (lesson_id, course_id)
+WITH t1 AS (
+SELECT 
+	lesson_id,
+	course_id,
+	COUNT(*) AS total_views,
+	COUNT(DISTINCT user_id) AS unique_users,
+	MIN(viewed_at)AS first_view,
+	MAX(viewed_at)AS last_view
+FROM core.fact_lesson_views
+GROUP BY lesson_id, course_id
+),
+t2 AS (
+SELECT 
+	l.lesson_id,
+	l.title AS lesson_title,
+	c.course_id,
+	c.title AS course_title
+FROM core.dim_course c 
+LEFT JOIN core.dim_lesson l USING(course_id)
+)
+
+SELECT 
+	t1.lesson_id,
+	t2.lesson_title,
+	t1.course_id,
+	t2.course_title,
+	t1.total_views,
+	t1.unique_users,
+	t1.first_view,
+	t1.last_view
+FROM t1
+LEFT JOIN t2 USING(lesson_id, course_id);
+
@@ -0,0 +1,28 @@
+/*
+ - ВИТРИНА АНАЛИЗА НЕАКТИВНЫХ ПОЛЬЗОВАТЕЛЕЙ
+ - ВСЕ УЧАСТВУЮЩИЕ ТАБЛИЦЫ DISTRIBUTED BY (user_id)
+ - ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ ТАК ЖЕ DISTRIBUTED BY (user_id)
+ */
+
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS dm.inactive_users_summary AS
+-- EXPLAIN ANALYZE
+DISTRIBUTED BY (user_id)
+WITH t1 AS (
+SELECT 
+	user_id,
+	count(DISTINCT course_id) AS enrollments_count
+FROM core.enrollments
+GROUP BY user_id
+)
+SELECT
+	u.user_id,
+	u.name AS user_name,
+	u.email,
+	u.age,
+	coalesce(t1.enrollments_count, 0) AS enrollments_count,
+	u.registration_date
+FROM core.dim_user u
+LEFT JOIN t1  USING (user_id)
+WHERE u.user_id NOT EXISTS (SELECT user_id FROM core.fact_lesson_views);
+
@@ -0,0 +1,40 @@
+/* 
+- ВИТРИНА ДЛЯ АНАЛИЗА КОНВЕРСИИ ПРОХОЖДЕНИЯ КУРСА ПОЛЬЗОВАТЕЛЯМИ
+- CORE.DIM_USER И CORE.FACT_LESSON_VIEWS DISTRIBUTED BY (user_id)
+- ИТОГОВОЕ РАСПРЕДЕЛЕНИЕ ПО (user_id, course_id)
+*/
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS dm.course_completion_rate AS
+WITH t1 AS (
+SELECT
+	e.user_id,
+	du.name AS user_name,
+	e.course_id, 
+	dc.title AS course_title,
+	COUNT(DISTINCT dl.lesson_id) AS lessons_in_course
+FROM core.enrollments e
+JOIN core.dim_user du USING (user_id)
+JOIN core.dim_course dc USING (course_id)
+LEFT JOIN core.dim_lesson dl USING (course_id)
+GROUP BY user_id, du.name, course_id, dc.title
+),
+t2 AS (
+SELECT 
+	user_id,
+	course_id,
+	COUNT(DISTINCT lesson_id) AS lessons_viewed
+FROM core.fact_lesson_views
+GROUP BY user_id, course_id
+)
+
+SELECT 
+	t1.user_id,
+	t1.user_name,
+	t1.course_id,
+	t1.course_title,
+	t1.lessons_in_course,
+	COALESCE(t2.lessons_viewed, 0) AS lessons_viewed,
+	COALESCE(ROUND(lessons_viewed::DECIMAL / NULLIF(lessons_in_course, 0), 2), 0) AS completion_rate
+FROM t1 
+LEFT JOIN t2 USING (user_id, course_id)
+DISTRIBUTED BY (user_id, course_id);
@@ -0,0 +1,3 @@
+SELECT * FROM dm.lesson_popularity_summary LIMIT 10;
+SELECT * FROM dm.inactive_users_summary LIMIT 10;
+SELECT * FROM dm.course_completion_rate LIMIT 10;