From bb70e5a0bdfa464decf120c137fe650688b6e079 Mon Sep 17 00:00:00 2001 From: dr0zd Date: Fri, 6 Feb 2026 06:50:41 +0700 Subject: [PATCH] finish pet-project clickhouse --- mdrozdov/clickhouse/stage2.sql | 143 +++++++++++++++++-------------- mdrozdov/clickhouse/stage3-4.sql | 45 ++++++++++ 2 files changed, 125 insertions(+), 63 deletions(-) create mode 100644 mdrozdov/clickhouse/stage3-4.sql diff --git a/mdrozdov/clickhouse/stage2.sql b/mdrozdov/clickhouse/stage2.sql index 1c74ed8..ec43feb 100644 --- a/mdrozdov/clickhouse/stage2.sql +++ b/mdrozdov/clickhouse/stage2.sql @@ -16,21 +16,27 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY (lesson_id, course_id); +-- здесь тригер таблица dim_lesson, хотя это не совсем правильно, но мы и состояние не храним +-- по сути таблица тригер должна быть fact_lesson_views, но right join будет плохо +-- а если не делать right join, то можем потерять логику, потому что left join от fact_lesson_view +-- логически теряет данные, если урок не был просмотрен +-- но оно хотя бы работает create materialized view mdrozdov.lesson_popularity_summary +on CLUSTER clickhouse_cluster to mdrozdov.dm_lesson_popularity_summary as select - dl.lesson_id as lesson_id, - dl.title as lesson_title, - dl.course_id as course_id, - dc.title as course_title, - count(flv.lesson_id) as total_views, - count(distinct flv.user_id) as unique_users, - min(flv.viewed_at) as first_view, - max(flv.viewed_at) as last_view -from mdrozdov.core_fact_lesson_views flv -right join mdrozdov.core_dim_course dc - on flv.course_id = dc.course_id -left join mdrozdov.core_dim_lesson dl + dl.lesson_id as lesson_id, + dl.title as lesson_title, + dl.course_id as course_id, + dc.title as course_title, + count(flv.lesson_id) as total_views, + count(distinct flv.user_id) as unique_users, + min(flv.viewed_at) as first_view, + max(flv.viewed_at) as last_view +from mdrozdov.core_dim_lesson dl +left join mdrozdov.core_dim_course dc + on dl.course_id = dc.course_id +left join mdrozdov.core_fact_lesson_views flv on dl.lesson_id = flv.lesson_id group by dl.lesson_id, dl.title, dl.course_id, dc.title @@ -50,23 +56,26 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY user_id +-- здесь тригер таблица dim_user +-- но почему-то при вставке в dim_user новые строки в таргет таблицу не добавляются create materialized view mdrozdov.inactive_users_summary - to mdrozdov.dm_inactive_users_summary - as - select - du.user_id as user_id, - du.name as name, - du.email as email, - du.age as age, - count(e.course_id) as registered_courses_count, - du.registration_date as registration_date - from mdrozdov.core_fact_lesson_views flv - right join mdrozdov.core_dim_user du - on du.user_id = flv.user_id - left join mdrozdov.core_enrollments e - on du.user_id = e.user_id - where flv.user_id is null - group by du.user_id, du.name, du.email, du.age, du.registration_date +on CLUSTER clickhouse_cluster +to mdrozdov.dm_inactive_users_summary +as +select + du.user_id as user_id, + du.name as name, + du.email as email, + du.age as age, + count(e.course_id) as registered_courses_count, + du.registration_date as registration_date +from mdrozdov.core_dim_user du +left join mdrozdov.core_enrollments e + on du.user_id = e.user_id +left join mdrozdov.core_fact_lesson_views flv + on du.user_id = flv.user_id +where flv.user_id is null +group by du.user_id, du.name, du.email, du.age, du.registration_date -- course_completion_rate create table mdrozdov.dm_course_completion_rate @@ -85,42 +94,44 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY (user_id, course_id) +-- здесь тригер таблица core_enrollments и тут все четко по логике и в работе create materialized view mdrozdov.course_completion_rate - to mdrozdov.dm_course_completion_rate as - with total_lessons_per_course as( - select - course_id, - count(*) as lessons_in_course - from mdrozdov.core_dim_lesson - group by course_id - ), lessons_viewed as ( - select - user_id, - course_id, - count(*) as lessons_viewed - from mdrozdov.core_fact_lesson_views flv - group by user_id, course_id - ) +on CLUSTER clickhouse_cluster +to mdrozdov.dm_course_completion_rate as +with total_lessons_per_course as( + select + course_id, + count(*) as lessons_in_course + from mdrozdov.core_dim_lesson + group by course_id +), lessons_viewed as ( select - du.user_id as user_id, - du.name as user_name, - dc.course_id as course_id, - dc.title as course_title, - tlpc.lessons_in_course as lessons_in_course, - coalesce(lv.lessons_viewed, 0) as lessons_viewed, - round( - coalesce( - coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 - ), 2) as completion_rate - from mdrozdov.core_enrollments e - join mdrozdov.core_dim_user du - on e.user_id = du.user_id - join total_lessons_per_course tlpc - on tlpc.course_id = e.course_id - join mdrozdov.core_dim_course dc - on e.course_id = dc.course_id - left join lessons_viewed lv - on du.user_id = lv.user_id and dc.course_id = lv.course_id + user_id, + course_id, + count(*) as lessons_viewed + from mdrozdov.core_fact_lesson_views flv + group by user_id, course_id +) +select + du.user_id as user_id, + du.name as user_name, + dc.course_id as course_id, + dc.title as course_title, + tlpc.lessons_in_course as lessons_in_course, + coalesce(lv.lessons_viewed, 0) as lessons_viewed, + round( + coalesce( + coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 + ), 2) as completion_rate +from mdrozdov.core_enrollments e +join mdrozdov.core_dim_user du + on e.user_id = du.user_id +join total_lessons_per_course tlpc + on tlpc.course_id = e.course_id +join mdrozdov.core_dim_course dc + on e.course_id = dc.course_id +left join lessons_viewed lv + on du.user_id = lv.user_id and dc.course_id = lv.course_id --данные из-за несовпадения названий столбцов в импорте/экспорте пришлось удалять, короче проще вручную -- их добавить @@ -132,4 +143,10 @@ insert into mdrozdov.core_fact_lesson_views(user_id, lesson_id, course_id, viewe (4, 4, 3, '2023-04-02 10:00:00'), (4, 5, 3, '2023-04-03 10:00:00') --- данные в mdrozdov.core_enrollments загружал через импорт \ No newline at end of file +insert into enrollments (user_id, course_id, enrolled_at) values +(1, 1, '2023-01-15'), +(1, 2, '2023-02-01'), +(2, 1, '2023-01-20'), +(3, 2, '2023-03-05'), +(4, 3, '2023-04-01') + diff --git a/mdrozdov/clickhouse/stage3-4.sql b/mdrozdov/clickhouse/stage3-4.sql new file mode 100644 index 0000000..dc80643 --- /dev/null +++ b/mdrozdov/clickhouse/stage3-4.sql @@ -0,0 +1,45 @@ +-- Топ-5 уроков с наименьшим количеством уникальных пользователей +SELECT + lesson_title, + course_title, + unique_users +FROM lesson_popularity_summary +ORDER BY unique_users ASC +LIMIT 5; + +-- Пользователи без зарегистрированных курсов +SELECT + name, + email, + registration_date +FROM inactive_users_summary +WHERE registered_courses_count = 0 +ORDER BY registration_date ASC; + +-- Список курсов с минимальной средней степенью завершения +SELECT + course_title, + AVG(completion_rate) AS avg_completion +FROM course_completion_rate +GROUP BY course_title +ORDER BY avg_completion ASC +LIMIT 5; + +-- Мини отчет: +/* +Таблица lesson_popularity_summary + Выбранные поля для проверки: + total_views - агрегат count + first_view - агрегат min + оба совпадают +Таблица inactive_users_summary + Выбранные поля для проверки: + registered_courses_count - агрегат count + совпадает +Таблица course_completion_rate + Выбранные поля для проверки: + completion_rate - агрегат count + совпадает +Итог: в моем случае все значения совпали, возможно если бы это было не через простой MergeTree +все было бы хуже +*/ \ No newline at end of file