diff --git a/ygurbanberdiev/clickhouse/Pet_Project/Pet_project_clickhouse.sql b/ygurbanberdiev/clickhouse/Pet_Project/Pet_project_clickhouse.sql new file mode 100644 index 0000000..4ea67ca --- /dev/null +++ b/ygurbanberdiev/clickhouse/Pet_Project/Pet_project_clickhouse.sql @@ -0,0 +1,287 @@ +-- Clickhouse. Пет-проект: Витринный слой +CREATE TABLE core_dim_user +ON CLUSTER clickhouse_cluster ( + user_id UInt32, + name String, + age UInt8, + email String, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_user', + '{replica}' +) +ORDER BY user_id; + + +CREATE TABLE core_dim_course +ON CLUSTER clickhouse_cluster ( + course_id UInt32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_course', + '{replica}' +) +ORDER BY course_id; + + +CREATE TABLE core_dim_lesson +ON CLUSTER clickhouse_cluster ( + lesson_id UInt32, + title String, + duration_min UInt8, + course_id UInt32 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_lesson', + '{replica}' +) +ORDER BY (course_id, lesson_id); + + +CREATE TABLE core_dim_date +ON CLUSTER clickhouse_cluster ( + date Date, + year UInt16, + month UInt8, + weekday String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_date', + '{replica}' +) +ORDER BY date; + + +CREATE TABLE core_fact_lesson_views +ON CLUSTER clickhouse_cluster ( + user_id UInt32, + lesson_id UInt32, + course_id UInt32, + viewed_at DateTime64(0) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/core_fact_lesson_views', + '{replica}' +) +ORDER BY (user_id, course_id, viewed_at, lesson_id); + + +CREATE TABLE core_enrollments +ON CLUSTER clickhouse_cluster ( + user_id UInt32, + course_id UInt32, + enrolled_at DateTime64(0) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/core_enrollments', + '{replica}' +) +ORDER BY (user_id, course_id); + + + +-- Этап 2. Витрина lesson_popularity_summary +CREATE TABLE lesson_popularity_summary +ON CLUSTER clickhouse_cluster ( + lesson_id UInt32, + lesson_title String, + course_id UInt32, + course_title String, + total_views UInt64, + unique_users UInt64, + first_view DateTime64(0), + last_view DateTime64(0) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/lesson_popularity_summary', + '{replica}' +) +ORDER BY (lesson_id, course_id); + +CREATE MATERIALIZED VIEW mv_lesson_popularity_summary +ON CLUSTER clickhouse_cluster +TO lesson_popularity_summary AS +WITH agg_table AS ( +SELECT + lesson_id, + course_id, + count() AS total_views, + uniqExact(user_id) AS unique_users, + min(viewed_at) AS first_view, + max(viewed_at) AS last_view +FROM core_fact_lesson_views +GROUP BY lesson_id, course_id +) +SELECT + atab.lesson_id AS lesson_id, + cdl.title AS lesson_title, + atab.course_id AS course_id, + cdc.title AS course_title, + atab.total_views AS total_views, + atab.unique_users AS unique_users, + atab.first_view AS first_view, + atab.last_view AS last_view +FROM agg_table AS atab +JOIN core_dim_lesson AS cdl ON atab.lesson_id = cdl.lesson_id +JOIN core_dim_course AS cdc ON atab.course_id = cdc.course_id; + + +-- Этап 2. Витрина inactive_users_summary +CREATE TABLE inactive_users_summary +ON CLUSTER clickhouse_cluster ( + user_id UInt64, + user_name String, + email String, + age UInt8, + registered_courses_count UInt64, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/inactive_users_summary', + '{replica}' +) +ORDER BY user_id; + + +CREATE MATERIALIZED VIEW mv_inactive_users_summary +ON CLUSTER clickhouse_cluster +TO inactive_users_summary +AS +SELECT + u.user_id AS user_id, + u.name AS user_name, + u.email AS email, + u.age AS age, + COALESCE(e.registered_courses_count, 0) AS registered_courses_count, + u.registration_date AS registration_date +FROM core_dim_user AS u +LEFT JOIN +( SELECT + user_id, + count() AS registered_courses_count + FROM core_enrollments + GROUP BY user_id +) AS e ON u.user_id = e.user_id +LEFT JOIN +( SELECT DISTINCT + user_id + FROM core_fact_lesson_views +) AS v ON u.user_id = v.user_id +WHERE v.user_id IS NULL; + + + +-- Этап 2. Витрина course_completion_rate +CREATE TABLE course_completion_rate +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + user_name String, + course_id UInt32, + course_title String, + lessons_in_course UInt64, + lessons_viewed UInt64, + completion_rate Float64 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ygurbanberdiev/course_completion_rate', + '{replica}' +) +ORDER BY (user_id, course_id); + +CREATE MATERIALIZED VIEW mv_course_completion_rate +ON CLUSTER clickhouse_cluster +TO course_completion_rate +AS +SELECT + e.user_id AS user_id, + u.name AS user_name, + e.course_id AS course_id, + c.title AS course_title, + COALESCE(lc.lessons_in_course, 0) AS lessons_in_course, + COALESCE(lv.lessons_viewed, 0) AS lessons_viewed, + if(COALESCE(lc.lessons_in_course, 0) = 0,0, round(COALESCE(lv.lessons_viewed, 0) * 1.0 / lc.lessons_in_course, 2)) AS completion_rate +FROM core_enrollments AS e +LEFT JOIN core_dim_user AS u ON e.user_id = u.user_id +LEFT JOIN core_dim_course AS c ON e.course_id = c.course_id +LEFT JOIN +( SELECT + course_id, + count() AS lessons_in_course + FROM core_dim_lesson + GROUP BY course_id +) AS lc ON e.course_id = lc.course_id +LEFT JOIN +( SELECT + user_id, + course_id, + uniqExact(lesson_id) AS lessons_viewed + FROM core_fact_lesson_views + GROUP BY user_id, course_id +) AS lv ON e.user_id = lv.user_id + AND e.course_id = lv.course_id; + + + +-- Этап 3. Уроки которые почти никто не смотрел +SELECT + lesson_title, + course_title, + total_views, + unique_users +FROM lesson_popularity_summary +WHERE total_views <= 1 +ORDER BY total_views ASC; + +-- Неактивные пользователи без записей на курсы +SELECT + user_name, + email, + age, + registration_date +FROM inactive_users_summary +WHERE registered_courses_count = 0 +ORDER BY registration_date DESC; + +-- Пользователи, полностью завершившие курс +SELECT + user_name, + course_title, + lessons_in_course, + lessons_viewed, + completion_rate +FROM course_completion_rate +WHERE completion_rate = 1 +ORDER BY course_title, user_name; + +/* Отчет 1: При сравнении витрины lesson_popularity_summary в ClickHouse и Greenplum результаты совпали только частично +Совпали показатели: +SUM(total_views) = 58 +MAX(total_views) = 9 + +Не совпали показатели: +AVG(total_views) +MIN(total_views) + +В Clickhouse: +AVG(total_views) = 3.2222 +MIN(total_views) = 0 +В Greenplum: +AVG(total_views) = 3.8666 +MIN(total_views) = 1 + +Причина расхождения: в ClickHouse в витрине есть дополнительные строки с total_views = 0, а в Greenplum таких строк нет. Из-за этих нулевых строк минимальное значение в ClickHouse стало 0, а среднее значение уменьшилось + + +Отчет 2: При проверке витрины inactive_users_summary результаты в ClickHouse и Greenplum совпали полностью. +В обеих системах получены одинаковые значения: +inactive_users_count = 9 +sum_registered_courses = 17 +avg_registered_courses = 1.8888888889 +min_registered_courses = 1 +max_registered_courses = 3 + +*/ \ No newline at end of file