diff --git a/itikhomirov/clickhouse/Pet_project.sql b/itikhomirov/clickhouse/Pet_project.sql new file mode 100644 index 0000000..27c2967 --- /dev/null +++ b/itikhomirov/clickhouse/Pet_project.sql @@ -0,0 +1,418 @@ +-- Этап 1 +-- Задание 1 +-- Перенести данные как в пет-проекте с Greenplum + +CREATE TABLE itikhomirov.core_user +ON CLUSTER clickhouse_cluster +( + user_id Int32, + name String, + age Int32, + email String, + registration_date DateTime64(3, 'UTC') +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_user', + '{replica}' +) +ORDER BY user_id; + +ALTER TABLE itikhomirov.core_user +MODIFY COLUMN registration_date DateTime64(3); + +CREATE TABLE itikhomirov.core_lesson +ON CLUSTER clickhouse_cluster +( + lesson_id Int32, + title String, + duration_min Int32, + course_id Int32 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_lesson', + '{replica}' +) +ORDER BY lesson_id; + +CREATE TABLE itikhomirov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster +( + user_id Int32, + lesson_id Int32, + course_id Int32, + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views', + '{replica}' +) +ORDER BY user_id; + +ALTER TABLE itikhomirov.core_fact_lesson_views +MODIFY COLUMN viewed_at DateTime64(3); + +DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster; + +CREATE TABLE itikhomirov.core_enrollments +ON CLUSTER clickhouse_cluster +( + user_id Int32, + course_id Int32, + enrolled_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE itikhomirov.core_course +ON CLUSTER clickhouse_cluster +( + course_id Int32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_course', + '{replica}' +) +ORDER BY course_id; + +-- Этап 2. Создание витринных таблиц +-- Задание 2: Подготовь структуру витрины lesson_popularity_summary + +-- Создаем пустую таблицу lesson_popularity_summary(структуру витрины) + +CREATE TABLE itikhomirov.lesson_popularity_summary +ON CLUSTER clickhouse_cluster +( + lesson_id UInt32, + lesson_title String, + course_id UInt32, + course_title String, + total_views UInt32, + unique_users UInt32, + first_view DateTime64(3), + last_view DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary', + '{replica}' +) +ORDER BY (lesson_id, course_id); + +-- Создаем Materialized View + +DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary +ON CLUSTER clickhouse_cluster; + +CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary +ON CLUSTER clickhouse_cluster +TO itikhomirov.lesson_popularity_summary +AS +SELECT + flv.lesson_id AS lesson_id, + l.title AS lesson_title, + flv.course_id AS course_id, + c.title AS course_title, + count() AS total_views, + uniqExact(flv.user_id) AS unique_users, + min(flv.viewed_at) AS first_view, + max(flv.viewed_at) AS last_view +FROM itikhomirov.core_fact_lesson_views AS flv +JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id +GROUP BY + flv.lesson_id, + l.title, + flv.course_id, + c.title; + +-- Вставляем данные в таблицу lesson_popularity_summary + +INSERT INTO itikhomirov.lesson_popularity_summary +SELECT + flv.lesson_id, + l.title AS lesson_title, + flv.course_id, + c.title AS course_title, + count() AS total_views, + uniqExact(flv.user_id) AS unique_users, + min(flv.viewed_at) AS first_view, + max(flv.viewed_at) AS last_view +FROM itikhomirov.core_fact_lesson_views AS flv +JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id +GROUP BY + flv.lesson_id, + l.title, + flv.course_id, + c.title; + +-- Задание 3: Создай витрину inactive_users_summary + +-- Создаем пустую таблицу inactive_users_summary(структуру витрины) + +DROP TABLE IF EXISTS itikhomirov.inactive_users_summary +ON CLUSTER clickhouse_cluster; + +DROP TABLE IF EXISTS itikhomirov.inactive_users_summary +ON CLUSTER clickhouse_cluster; + +CREATE TABLE itikhomirov.inactive_users_summary +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + name String, + email String, + age UInt8, + registered_courses_count UInt32, + registration_date DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary', + '{replica}' +) +ORDER BY (user_id); + +-- Создаем Materialized View + +DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary +ON CLUSTER clickhouse_cluster; + +CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary +ON CLUSTER clickhouse_cluster +TO itikhomirov.inactive_users_summary +AS +WITH user_courses AS ( + SELECT + e.user_id, + countDistinct(e.course_id) AS registered_courses_count + FROM itikhomirov.core_enrollments AS e + GROUP BY + e.user_id +) +SELECT + u.user_id AS user_id, + u.name AS name, + u.email AS email, + u.age AS age, + coalesce(uc.registered_courses_count, 0) AS registered_courses_count, + u.registration_date AS registration_date +FROM itikhomirov.core_user AS u +LEFT JOIN user_courses AS uc + ON u.user_id = uc.user_id +WHERE u.user_id NOT IN ( + SELECT DISTINCT user_id + FROM itikhomirov.core_fact_lesson_views +); + + +INSERT INTO itikhomirov.inactive_users_summary +WITH user_courses AS ( + SELECT + e.user_id, + countDistinct(e.course_id) AS registered_courses_count + FROM itikhomirov.core_enrollments AS e + GROUP BY + e.user_id +) +-- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров +-- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views), +-- и к ним подмешивается уже готовый registered_courses_count. +SELECT + u.user_id AS user_id, + u.name AS name, + u.email AS email, + u.age AS age, + coalesce(uc.registered_courses_count, 0) AS registered_courses_count, + u.registration_date AS registration_date +FROM itikhomirov.core_user AS u +LEFT JOIN user_courses AS uc + ON u.user_id = uc.user_id +WHERE u.user_id NOT IN ( + SELECT DISTINCT user_id + FROM itikhomirov.core_fact_lesson_views +) +ORDER BY u.user_id; + +-- Задание 4: Создай витрину course_completion_rate + +CREATE TABLE itikhomirov.course_completion_rate +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + user_name String, + course_id UInt32, + course_title String, + lessons_in_course UInt32, + lessons_viewed UInt32, + completion_rate Float32 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/course_completion_rate', + '{replica}' +) +ORDER BY (user_id, course_id); + + +-- Создаем Materialized View + +CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate +ON CLUSTER clickhouse_cluster +TO itikhomirov.course_completion_rate +AS +WITH +lessons_per_course AS ( + SELECT + course_id, + countDistinct(lesson_id) AS lessons_in_course + FROM itikhomirov.core_lesson + GROUP BY course_id +), +views_per_user_course AS ( + SELECT + user_id, + course_id, + countDistinct(lesson_id) AS lessons_viewed + FROM itikhomirov.core_fact_lesson_views + GROUP BY user_id, course_id +) +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + lpc.lessons_in_course AS lessons_in_course, + coalesce(vpc.lessons_viewed, 0) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0) + / NULLIF(lpc.lessons_in_course * 1.0, 0), + 0 + ), + 2 + ) AS completion_rate +FROM itikhomirov.core_user AS u +JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id +JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id +JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id +LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id +ORDER BY + u.user_id, + c.course_id; + +-- Вставляем данные в таблицу pet.course_completion_rate + +INSERT INTO itikhomirov.course_completion_rate +-- Создаем две временные таблицы: +-- 1) Сколько уроков в каждом курсе +WITH +lessons_per_course AS ( + SELECT + course_id, + countDistinct(lesson_id) AS lessons_in_course + FROM itikhomirov.core_lesson + GROUP BY course_id +), +-- 2) Сколько уроков каждого курса посмотрел пользователь +views_per_user_course AS ( + SELECT + user_id, + course_id, + countDistinct(lesson_id) AS lessons_viewed + FROM itikhomirov.core_fact_lesson_views + GROUP BY user_id, course_id +) +-- Основной запрос +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + lpc.lessons_in_course AS lessons_in_course, + coalesce(vpc.lessons_viewed, 0) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0) + / NULLIF(lpc.lessons_in_course * 1.0, 0), + 0 + ), + 2 + ) AS completion_rate +FROM itikhomirov.core_user AS u +JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id +JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id +JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id +LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id +ORDER BY + u.user_id, + c.course_id; + +-- Этап 3. Анализ и запросы + +-- Задание 5: Напиши SQL-запросы к витринам + +-- Уроки с хотя бы одним просмотром +SELECT + lesson_title, + course_title, + total_views +FROM itikhomirov.lesson_popularity_summary +WHERE total_views > 0 +ORDER BY lesson_title ASC; + +-- Топ‑3 урока внутри одного курса +SELECT + lesson_title, + course_title, + total_views +FROM itikhomirov.lesson_popularity_summary +WHERE course_title = 'SQL для начинающих' +ORDER BY total_views DESC +LIMIT 3; + +-- Пользователи, которые завершили курсы на 100% +SELECT + user_id, + user_name, + course_title, + completion_rate +FROM itikhomirov.course_completion_rate +WHERE completion_rate = 1 +ORDER BY user_id; + +-- Этап 4. Тестирование и валидация + +-- Задание 6: Сравни данные с Greenplum** + +-- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum + + +-- 2. Сравни с ClickHouse-результатами: +-- - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX` +-- - Проверяй `NULL`, `0`, округления + + +-- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"** + +SELECT + sum(total_views) AS sum_total_views, + avg(total_views) AS avg_total_views, + min(total_views) AS min_total_views, + max(total_views) AS max_total_views +FROM itikhomirov.lesson_popularity_summary +WHERE lesson_id IN (1, 3, 5); + +SELECT + sum(total_views) AS sum_total_views, + avg(total_views) AS avg_total_views, + min(total_views) AS min_total_views, + max(total_views) AS max_total_views +FROM itikhomirov.lesson_popularity_summary +WHERE lesson_id = 999; + +-- Отчёт в .doc файле + diff --git a/itikhomirov/clickhouse/Pet_project_Step 1.1.png b/itikhomirov/clickhouse/Pet_project_Step 1.1.png new file mode 100644 index 0000000..82fc7ae Binary files /dev/null and b/itikhomirov/clickhouse/Pet_project_Step 1.1.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 2.2.png b/itikhomirov/clickhouse/Pet_project_Step 2.2.png new file mode 100644 index 0000000..81d64b7 Binary files /dev/null and b/itikhomirov/clickhouse/Pet_project_Step 2.2.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 2.3.png b/itikhomirov/clickhouse/Pet_project_Step 2.3.png new file mode 100644 index 0000000..b3cc847 Binary files /dev/null and b/itikhomirov/clickhouse/Pet_project_Step 2.3.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 2.4.png b/itikhomirov/clickhouse/Pet_project_Step 2.4.png new file mode 100644 index 0000000..237d974 Binary files /dev/null and b/itikhomirov/clickhouse/Pet_project_Step 2.4.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 3.5.png b/itikhomirov/clickhouse/Pet_project_Step 3.5.png new file mode 100644 index 0000000..ed6d534 Binary files /dev/null and b/itikhomirov/clickhouse/Pet_project_Step 3.5.png differ diff --git a/itikhomirov/clickhouse/Step 4.6.pdf b/itikhomirov/clickhouse/Step 4.6.pdf new file mode 100644 index 0000000..804ddc2 Binary files /dev/null and b/itikhomirov/clickhouse/Step 4.6.pdf differ