Merge pull request 'add clickhouse pet project' (#211) from DEIN-1773 into main

Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/211 Reviewed-by: aleksey bogatyrev <abogatyrev@example.com>
2026-06-11 03:40:43 +04:00
parent ffdac5446e acc0352b81
commit 84332f0313
1 changed files with 287 additions and 0 deletions
@@ -0,0 +1,287 @@
 -- Clickhouse. Пет-проект: Витринный слой
 CREATE TABLE core_dim_user 
 ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    name String,
    age UInt8,
    email String,
    registration_date Date
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_user',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE TABLE core_dim_course 
 ON CLUSTER clickhouse_cluster (
    course_id UInt32,
    title String,
    category String
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_course',
    '{replica}'
 )
 ORDER BY course_id;
 CREATE TABLE core_dim_lesson 
 ON CLUSTER clickhouse_cluster (
    lesson_id UInt32,
    title String,
    duration_min UInt8,
    course_id UInt32
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_lesson',
    '{replica}'
 )
 ORDER BY (course_id, lesson_id);
 CREATE TABLE core_dim_date 
 ON CLUSTER clickhouse_cluster (
    date Date,
    year UInt16,
    month UInt8,
    weekday String
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_date',
    '{replica}'
 )
 ORDER BY date;
 CREATE TABLE core_fact_lesson_views 
 ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    lesson_id UInt32,
    course_id UInt32,
    viewed_at DateTime64(0)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_fact_lesson_views',
    '{replica}'
 )
 ORDER BY (user_id, course_id, viewed_at, lesson_id);
 CREATE TABLE core_enrollments
 ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    course_id UInt32,
    enrolled_at DateTime64(0)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_enrollments',
    '{replica}'
 )
 ORDER BY (user_id, course_id);
 -- Этап 2. Витрина lesson_popularity_summary
 CREATE TABLE lesson_popularity_summary
 ON CLUSTER clickhouse_cluster (
 	lesson_id UInt32,
 	lesson_title String,
 	course_id UInt32,
 	course_title String,
 	total_views UInt64,
 	unique_users UInt64,
 	first_view DateTime64(0),
 	last_view DateTime64(0)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/lesson_popularity_summary',
    '{replica}'
 )
 ORDER BY (lesson_id, course_id);
 CREATE MATERIALIZED VIEW mv_lesson_popularity_summary
 ON CLUSTER clickhouse_cluster
 TO lesson_popularity_summary AS 
 WITH agg_table AS (
 SELECT
 	lesson_id,
 	course_id,
 	count() AS total_views,
 	uniqExact(user_id) AS unique_users,
 	min(viewed_at) AS first_view,
 	max(viewed_at) AS last_view
 FROM core_fact_lesson_views
 GROUP BY lesson_id, course_id
 )
 SELECT
 	atab.lesson_id AS lesson_id,
 	cdl.title AS lesson_title,
 	atab.course_id AS course_id,
 	cdc.title AS course_title,
 	atab.total_views AS total_views,
 	atab.unique_users AS unique_users,
 	atab.first_view AS first_view,
 	atab.last_view AS last_view
 FROM agg_table AS atab
 JOIN core_dim_lesson AS cdl ON atab.lesson_id = cdl.lesson_id 
 JOIN core_dim_course AS cdc ON atab.course_id = cdc.course_id;
 -- Этап 2. Витрина inactive_users_summary
 CREATE TABLE inactive_users_summary
 ON CLUSTER clickhouse_cluster (
 	user_id UInt64,
 	user_name String,
 	email String,
 	age UInt8,
 	registered_courses_count UInt64,
 	registration_date Date
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/inactive_users_summary',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE MATERIALIZED VIEW mv_inactive_users_summary
 ON CLUSTER clickhouse_cluster
 TO inactive_users_summary
 AS
 SELECT
    u.user_id AS user_id,
    u.name AS user_name,
    u.email AS email,
    u.age AS age,
    COALESCE(e.registered_courses_count, 0) AS registered_courses_count,
    u.registration_date AS registration_date
 FROM core_dim_user AS u
 LEFT JOIN
 (	SELECT
 		user_id,
 		count() AS registered_courses_count
 	FROM core_enrollments
   GROUP BY user_id
 ) AS e ON u.user_id = e.user_id
 LEFT JOIN
 (	SELECT DISTINCT
   		user_id
 	FROM core_fact_lesson_views
 ) AS v ON u.user_id = v.user_id
 WHERE v.user_id IS NULL;
 -- Этап 2. Витрина course_completion_rate
 CREATE TABLE course_completion_rate
 ON CLUSTER clickhouse_cluster
 (
    user_id UInt32,
    user_name String,
    course_id UInt32,
    course_title String,
    lessons_in_course UInt64,
    lessons_viewed UInt64,
    completion_rate Float64
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/course_completion_rate',
    '{replica}'
 )
 ORDER BY (user_id, course_id);
 CREATE MATERIALIZED VIEW mv_course_completion_rate
 ON CLUSTER clickhouse_cluster
 TO course_completion_rate
 AS
 SELECT
 	e.user_id AS user_id,
 	u.name AS user_name,
 	e.course_id AS course_id,
 	c.title AS course_title,
 	COALESCE(lc.lessons_in_course, 0) AS lessons_in_course,
 	COALESCE(lv.lessons_viewed, 0) AS lessons_viewed,
 	if(COALESCE(lc.lessons_in_course, 0) = 0,0, round(COALESCE(lv.lessons_viewed, 0) * 1.0 / lc.lessons_in_course, 2)) AS completion_rate
 FROM core_enrollments AS e
 LEFT JOIN core_dim_user AS u ON e.user_id = u.user_id
 LEFT JOIN core_dim_course AS c ON e.course_id = c.course_id
 LEFT JOIN
 (	SELECT
 		course_id,
 		count() AS lessons_in_course
 	FROM core_dim_lesson
 	GROUP BY course_id
 ) AS lc ON e.course_id = lc.course_id
 LEFT JOIN
 (	SELECT
 		user_id,
 		course_id,
 		uniqExact(lesson_id) AS lessons_viewed
 	FROM core_fact_lesson_views
 	GROUP BY user_id, course_id
 ) AS lv ON e.user_id = lv.user_id
   AND e.course_id = lv.course_id;
 -- Этап 3. Уроки которые почти никто не смотрел
 SELECT
    lesson_title,
    course_title,
    total_views,
    unique_users
 FROM lesson_popularity_summary
 WHERE total_views <= 1
 ORDER BY total_views ASC;
 -- Неактивные пользователи без записей на курсы
 SELECT
    user_name,
    email,
    age,
    registration_date
 FROM inactive_users_summary
 WHERE registered_courses_count = 0
 ORDER BY registration_date DESC;
 -- Пользователи, полностью завершившие курс
 SELECT
    user_name,
    course_title,
    lessons_in_course,
    lessons_viewed,
    completion_rate
 FROM course_completion_rate
 WHERE completion_rate = 1
 ORDER BY course_title, user_name;
 /* Отчет 1: При сравнении витрины lesson_popularity_summary в ClickHouse и Greenplum результаты совпали только частично
 Совпали показатели:
 SUM(total_views) = 58
 MAX(total_views) = 9
 Не совпали показатели:
 AVG(total_views)
 MIN(total_views) 
 В Clickhouse:
 AVG(total_views) = 3.2222
 MIN(total_views) = 0
 В Greenplum: 
 AVG(total_views) = 3.8666
 MIN(total_views) = 1
 Причина расхождения: в ClickHouse в витрине есть дополнительные строки с total_views = 0, а в Greenplum таких строк нет. Из-за этих нулевых строк минимальное значение в ClickHouse стало 0, а среднее значение уменьшилось 
 Отчет 2: При проверке витрины inactive_users_summary результаты в ClickHouse и Greenplum совпали полностью.
 В обеих системах получены одинаковые значения:
 inactive_users_count = 9
 sum_registered_courses = 17
 avg_registered_courses = 1.8888888889
 min_registered_courses = 1
 max_registered_courses = 3
 */