diff --git a/mdrozdov/clickhouse/stage1.sql b/mdrozdov/clickhouse/stage1.sql new file mode 100644 index 0000000..5242dea --- /dev/null +++ b/mdrozdov/clickhouse/stage1.sql @@ -0,0 +1,96 @@ +-- удалить таблицу если раньше создавали +DROP TABLE IF EXISTS mdrozdov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster; + +-- таблицы измерений +-- user dimension +CREATE TABLE mdrozdov.core_dim_user +ON CLUSTER clickhouse_cluster +( + user_id Int32, + name String, + age Int32, + email String, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_user', + '{replica}' +) +ORDER BY user_id; + +-- course dimension +CREATE TABLE mdrozdov.core_dim_course +ON CLUSTER clickhouse_cluster +( + course_id Int32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_course', + '{replica}' +) +ORDER BY course_id; + +-- lesson dimension +CREATE TABLE mdrozdov.core_dim_lesson +ON CLUSTER clickhouse_cluster +( + lesson_id Int32, + title String, + duration_min Int32, + course_id Int32, +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_lesson2', + '{replica}' +) +ORDER BY (course_id, lesson_id); + +-- date dimension +CREATE TABLE mdrozdov.core_dim_date +ON CLUSTER clickhouse_cluster +( + date DATE, + year Int32, + month Int32, + day Int32, + weekday String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_date', + '{replica}' +) +ORDER BY date; + + +-- таблицы фактов +-- core_enrollments +CREATE TABLE mdrozdov.core_enrollments +ON CLUSTER clickhouse_cluster +( + user_id Int32, + course_id Int32, + enrolled_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_enrollments', + '{replica}' +) +ORDER BY course_id; + +-- core_fact_lesson_views +CREATE TABLE mdrozdov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster +( + user_id Int32, + lesson_id Int32, + course_id Int32, + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_fact_lesson_views', + '{replica}' +) +ORDER BY course_id; diff --git a/mdrozdov/clickhouse/stage2.sql b/mdrozdov/clickhouse/stage2.sql new file mode 100644 index 0000000..ec43feb --- /dev/null +++ b/mdrozdov/clickhouse/stage2.sql @@ -0,0 +1,152 @@ +--lesson_popularity_summary data mart +create table mdrozdov.dm_lesson_popularity_summary +ON CLUSTER clickhouse_cluster( + lesson_id Int32, + lesson_title String, + course_id Int32, + course_title String, + total_views Int32, + unique_users Int32, + first_view DateTime64(3), + last_view DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/dm_lesson_popularity_summary1', + '{replica}' +) +ORDER BY (lesson_id, course_id); + +-- здесь тригер таблица dim_lesson, хотя это не совсем правильно, но мы и состояние не храним +-- по сути таблица тригер должна быть fact_lesson_views, но right join будет плохо +-- а если не делать right join, то можем потерять логику, потому что left join от fact_lesson_view +-- логически теряет данные, если урок не был просмотрен +-- но оно хотя бы работает +create materialized view mdrozdov.lesson_popularity_summary +on CLUSTER clickhouse_cluster +to mdrozdov.dm_lesson_popularity_summary as +select + dl.lesson_id as lesson_id, + dl.title as lesson_title, + dl.course_id as course_id, + dc.title as course_title, + count(flv.lesson_id) as total_views, + count(distinct flv.user_id) as unique_users, + min(flv.viewed_at) as first_view, + max(flv.viewed_at) as last_view +from mdrozdov.core_dim_lesson dl +left join mdrozdov.core_dim_course dc + on dl.course_id = dc.course_id +left join mdrozdov.core_fact_lesson_views flv + on dl.lesson_id = flv.lesson_id +group by dl.lesson_id, dl.title, dl.course_id, dc.title + +--inactive_users_summary +create table mdrozdov.dm_inactive_users_summary +ON CLUSTER clickhouse_cluster( + user_id Int32, + name String, + email String, + age Int32, + registered_courses_count Int32, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/dm_inactive_users_summary1', + '{replica}' +) +ORDER BY user_id + +-- здесь тригер таблица dim_user +-- но почему-то при вставке в dim_user новые строки в таргет таблицу не добавляются +create materialized view mdrozdov.inactive_users_summary +on CLUSTER clickhouse_cluster +to mdrozdov.dm_inactive_users_summary +as +select + du.user_id as user_id, + du.name as name, + du.email as email, + du.age as age, + count(e.course_id) as registered_courses_count, + du.registration_date as registration_date +from mdrozdov.core_dim_user du +left join mdrozdov.core_enrollments e + on du.user_id = e.user_id +left join mdrozdov.core_fact_lesson_views flv + on du.user_id = flv.user_id +where flv.user_id is null +group by du.user_id, du.name, du.email, du.age, du.registration_date + +-- course_completion_rate +create table mdrozdov.dm_course_completion_rate +ON CLUSTER clickhouse_cluster( + user_id Int32, + user_name String, + course_id Int32, + course_title String, + lessons_in_course Int32, + lessons_viewed Int32, + completion_rate Decimal(4, 3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/dm_course_completion_rate', + '{replica}' +) +ORDER BY (user_id, course_id) + +-- здесь тригер таблица core_enrollments и тут все четко по логике и в работе +create materialized view mdrozdov.course_completion_rate +on CLUSTER clickhouse_cluster +to mdrozdov.dm_course_completion_rate as +with total_lessons_per_course as( + select + course_id, + count(*) as lessons_in_course + from mdrozdov.core_dim_lesson + group by course_id +), lessons_viewed as ( + select + user_id, + course_id, + count(*) as lessons_viewed + from mdrozdov.core_fact_lesson_views flv + group by user_id, course_id +) +select + du.user_id as user_id, + du.name as user_name, + dc.course_id as course_id, + dc.title as course_title, + tlpc.lessons_in_course as lessons_in_course, + coalesce(lv.lessons_viewed, 0) as lessons_viewed, + round( + coalesce( + coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 + ), 2) as completion_rate +from mdrozdov.core_enrollments e +join mdrozdov.core_dim_user du + on e.user_id = du.user_id +join total_lessons_per_course tlpc + on tlpc.course_id = e.course_id +join mdrozdov.core_dim_course dc + on e.course_id = dc.course_id +left join lessons_viewed lv + on du.user_id = lv.user_id and dc.course_id = lv.course_id + +--данные из-за несовпадения названий столбцов в импорте/экспорте пришлось удалять, короче проще вручную +-- их добавить +insert into mdrozdov.core_fact_lesson_views(user_id, lesson_id, course_id, viewed_at) values +(1, 1, 1, '2023-01-16 10:00:00'), +(1, 2, 1, '2023-01-16 10:15:00'), +(2, 1, 1, '2023-01-21 09:00:00'), +(3, 3, 2, '2023-03-06 12:00:00'), +(4, 4, 3, '2023-04-02 10:00:00'), +(4, 5, 3, '2023-04-03 10:00:00') + +insert into enrollments (user_id, course_id, enrolled_at) values +(1, 1, '2023-01-15'), +(1, 2, '2023-02-01'), +(2, 1, '2023-01-20'), +(3, 2, '2023-03-05'), +(4, 3, '2023-04-01') + diff --git a/mdrozdov/clickhouse/stage3-4.sql b/mdrozdov/clickhouse/stage3-4.sql new file mode 100644 index 0000000..dc80643 --- /dev/null +++ b/mdrozdov/clickhouse/stage3-4.sql @@ -0,0 +1,45 @@ +-- Топ-5 уроков с наименьшим количеством уникальных пользователей +SELECT + lesson_title, + course_title, + unique_users +FROM lesson_popularity_summary +ORDER BY unique_users ASC +LIMIT 5; + +-- Пользователи без зарегистрированных курсов +SELECT + name, + email, + registration_date +FROM inactive_users_summary +WHERE registered_courses_count = 0 +ORDER BY registration_date ASC; + +-- Список курсов с минимальной средней степенью завершения +SELECT + course_title, + AVG(completion_rate) AS avg_completion +FROM course_completion_rate +GROUP BY course_title +ORDER BY avg_completion ASC +LIMIT 5; + +-- Мини отчет: +/* +Таблица lesson_popularity_summary + Выбранные поля для проверки: + total_views - агрегат count + first_view - агрегат min + оба совпадают +Таблица inactive_users_summary + Выбранные поля для проверки: + registered_courses_count - агрегат count + совпадает +Таблица course_completion_rate + Выбранные поля для проверки: + completion_rate - агрегат count + совпадает +Итог: в моем случае все значения совпали, возможно если бы это было не через простой MergeTree +все было бы хуже +*/ \ No newline at end of file