Merge pull request 'add clickhouse pet project' (#211) from DEIN-1773 into main

Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/211 Reviewed-by: aleksey bogatyrev <abogatyrev@example.com>
2026-06-11 03:40:43 +04:00
parent ffdac5446e acc0352b81
commit 84332f0313
1 changed files with 287 additions and 0 deletions
@@ -0,0 +1,287 @@
+-- Clickhouse. Пет-проект: Витринный слой
+CREATE TABLE core_dim_user 
+ON CLUSTER clickhouse_cluster (
+    user_id UInt32,
+    name String,
+    age UInt8,
+    email String,
+    registration_date Date
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_user',
+    '{replica}'
+)
+ORDER BY user_id;
+
+
+CREATE TABLE core_dim_course 
+ON CLUSTER clickhouse_cluster (
+    course_id UInt32,
+    title String,
+    category String
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_course',
+    '{replica}'
+)
+ORDER BY course_id;
+
+
+CREATE TABLE core_dim_lesson 
+ON CLUSTER clickhouse_cluster (
+    lesson_id UInt32,
+    title String,
+    duration_min UInt8,
+    course_id UInt32
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_lesson',
+    '{replica}'
+)
+ORDER BY (course_id, lesson_id);
+
+
+CREATE TABLE core_dim_date 
+ON CLUSTER clickhouse_cluster (
+    date Date,
+    year UInt16,
+    month UInt8,
+    weekday String
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_date',
+    '{replica}'
+)
+ORDER BY date;
+
+
+CREATE TABLE core_fact_lesson_views 
+ON CLUSTER clickhouse_cluster (
+    user_id UInt32,
+    lesson_id UInt32,
+    course_id UInt32,
+    viewed_at DateTime64(0)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/core_fact_lesson_views',
+    '{replica}'
+)
+ORDER BY (user_id, course_id, viewed_at, lesson_id);
+
+
+CREATE TABLE core_enrollments
+ON CLUSTER clickhouse_cluster (
+    user_id UInt32,
+    course_id UInt32,
+    enrolled_at DateTime64(0)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/core_enrollments',
+    '{replica}'
+)
+ORDER BY (user_id, course_id);
+
+
+
+-- Этап 2. Витрина lesson_popularity_summary
+CREATE TABLE lesson_popularity_summary
+ON CLUSTER clickhouse_cluster (
+	lesson_id UInt32,
+	lesson_title String,
+	course_id UInt32,
+	course_title String,
+	total_views UInt64,
+	unique_users UInt64,
+	first_view DateTime64(0),
+	last_view DateTime64(0)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/lesson_popularity_summary',
+    '{replica}'
+)
+ORDER BY (lesson_id, course_id);
+
+CREATE MATERIALIZED VIEW mv_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster
+TO lesson_popularity_summary AS 
+WITH agg_table AS (
+SELECT
+	lesson_id,
+	course_id,
+	count() AS total_views,
+	uniqExact(user_id) AS unique_users,
+	min(viewed_at) AS first_view,
+	max(viewed_at) AS last_view
+FROM core_fact_lesson_views
+GROUP BY lesson_id, course_id
+)
+SELECT
+	atab.lesson_id AS lesson_id,
+	cdl.title AS lesson_title,
+	atab.course_id AS course_id,
+	cdc.title AS course_title,
+	atab.total_views AS total_views,
+	atab.unique_users AS unique_users,
+	atab.first_view AS first_view,
+	atab.last_view AS last_view
+FROM agg_table AS atab
+JOIN core_dim_lesson AS cdl ON atab.lesson_id = cdl.lesson_id 
+JOIN core_dim_course AS cdc ON atab.course_id = cdc.course_id;
+
+
+-- Этап 2. Витрина inactive_users_summary
+CREATE TABLE inactive_users_summary
+ON CLUSTER clickhouse_cluster (
+	user_id UInt64,
+	user_name String,
+	email String,
+	age UInt8,
+	registered_courses_count UInt64,
+	registration_date Date
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/inactive_users_summary',
+    '{replica}'
+)
+ORDER BY user_id;
+
+
+CREATE MATERIALIZED VIEW mv_inactive_users_summary
+ON CLUSTER clickhouse_cluster
+TO inactive_users_summary
+AS
+SELECT
+    u.user_id AS user_id,
+    u.name AS user_name,
+    u.email AS email,
+    u.age AS age,
+    COALESCE(e.registered_courses_count, 0) AS registered_courses_count,
+    u.registration_date AS registration_date
+FROM core_dim_user AS u
+LEFT JOIN
+(	SELECT
+		user_id,
+		count() AS registered_courses_count
+	FROM core_enrollments
+   GROUP BY user_id
+) AS e ON u.user_id = e.user_id
+LEFT JOIN
+(	SELECT DISTINCT
+   		user_id
+	FROM core_fact_lesson_views
+) AS v ON u.user_id = v.user_id
+WHERE v.user_id IS NULL;
+
+
+
+-- Этап 2. Витрина course_completion_rate
+CREATE TABLE course_completion_rate
+ON CLUSTER clickhouse_cluster
+(
+    user_id UInt32,
+    user_name String,
+    course_id UInt32,
+    course_title String,
+    lessons_in_course UInt64,
+    lessons_viewed UInt64,
+    completion_rate Float64
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/ygurbanberdiev/course_completion_rate',
+    '{replica}'
+)
+ORDER BY (user_id, course_id);
+
+CREATE MATERIALIZED VIEW mv_course_completion_rate
+ON CLUSTER clickhouse_cluster
+TO course_completion_rate
+AS
+SELECT
+	e.user_id AS user_id,
+	u.name AS user_name,
+	e.course_id AS course_id,
+	c.title AS course_title,
+	COALESCE(lc.lessons_in_course, 0) AS lessons_in_course,
+	COALESCE(lv.lessons_viewed, 0) AS lessons_viewed,
+	if(COALESCE(lc.lessons_in_course, 0) = 0,0, round(COALESCE(lv.lessons_viewed, 0) * 1.0 / lc.lessons_in_course, 2)) AS completion_rate
+FROM core_enrollments AS e
+LEFT JOIN core_dim_user AS u ON e.user_id = u.user_id
+LEFT JOIN core_dim_course AS c ON e.course_id = c.course_id
+LEFT JOIN
+(	SELECT
+		course_id,
+		count() AS lessons_in_course
+	FROM core_dim_lesson
+	GROUP BY course_id
+) AS lc ON e.course_id = lc.course_id
+LEFT JOIN
+(	SELECT
+		user_id,
+		course_id,
+		uniqExact(lesson_id) AS lessons_viewed
+	FROM core_fact_lesson_views
+	GROUP BY user_id, course_id
+) AS lv ON e.user_id = lv.user_id
+   AND e.course_id = lv.course_id;
+
+
+	
+-- Этап 3. Уроки которые почти никто не смотрел
+SELECT
+    lesson_title,
+    course_title,
+    total_views,
+    unique_users
+FROM lesson_popularity_summary
+WHERE total_views <= 1
+ORDER BY total_views ASC;
+
+-- Неактивные пользователи без записей на курсы
+SELECT
+    user_name,
+    email,
+    age,
+    registration_date
+FROM inactive_users_summary
+WHERE registered_courses_count = 0
+ORDER BY registration_date DESC;
+
+-- Пользователи, полностью завершившие курс
+SELECT
+    user_name,
+    course_title,
+    lessons_in_course,
+    lessons_viewed,
+    completion_rate
+FROM course_completion_rate
+WHERE completion_rate = 1
+ORDER BY course_title, user_name;
+
+/* Отчет 1: При сравнении витрины lesson_popularity_summary в ClickHouse и Greenplum результаты совпали только частично
+Совпали показатели:
+SUM(total_views) = 58
+MAX(total_views) = 9
+
+Не совпали показатели:
+AVG(total_views)
+MIN(total_views) 
+
+В Clickhouse:
+AVG(total_views) = 3.2222
+MIN(total_views) = 0
+В Greenplum: 
+AVG(total_views) = 3.8666
+MIN(total_views) = 1
+
+Причина расхождения: в ClickHouse в витрине есть дополнительные строки с total_views = 0, а в Greenplum таких строк нет. Из-за этих нулевых строк минимальное значение в ClickHouse стало 0, а среднее значение уменьшилось 
+
+
+Отчет 2: При проверке витрины inactive_users_summary результаты в ClickHouse и Greenplum совпали полностью.
+В обеих системах получены одинаковые значения:
+inactive_users_count = 9
+sum_registered_courses = 17
+avg_registered_courses = 1.8888888889
+min_registered_courses = 1
+max_registered_courses = 3
+
+*/