From d280456d0919ddc8f64382d0c7b16846ab8cb38b Mon Sep 17 00:00:00 2001 From: Bonifatsiy Date: Mon, 11 May 2026 04:21:02 +0400 Subject: [PATCH] =?UTF-8?q?=D0=BF=D0=B5=D1=82-=D0=BF=D1=80=D0=BE=D0=B5?= =?UTF-8?q?=D0=BA=D1=82=20=D0=BF=D0=BE=D0=BB=D0=BD=D0=BE=D1=81=D1=82=D1=8C?= =?UTF-8?q?=D1=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Этап 1. Загрузка тестовых данных.sql | 90 +++++++++ .../Этап 2. Создание витринных таблиц.sql | 186 ++++++++++++++++++ .../Pet-Project/Этап 3. Анализ и запросы.sql | 88 +++++++++ .../Этап 4. Тестирование и валидация.sql | 13 ++ 4 files changed, 377 insertions(+) create mode 100644 dsolovianov/clickhouse/Pet-Project/Этап 1. Загрузка тестовых данных.sql create mode 100644 dsolovianov/clickhouse/Pet-Project/Этап 2. Создание витринных таблиц.sql create mode 100644 dsolovianov/clickhouse/Pet-Project/Этап 3. Анализ и запросы.sql create mode 100644 dsolovianov/clickhouse/Pet-Project/Этап 4. Тестирование и валидация.sql diff --git a/dsolovianov/clickhouse/Pet-Project/Этап 1. Загрузка тестовых данных.sql b/dsolovianov/clickhouse/Pet-Project/Этап 1. Загрузка тестовых данных.sql new file mode 100644 index 0000000..167607d --- /dev/null +++ b/dsolovianov/clickhouse/Pet-Project/Этап 1. Загрузка тестовых данных.sql @@ -0,0 +1,90 @@ +DROP TABLE IF EXISTS dsolovianov.core_fact_lesson_views ON CLUSTER clickhouse_cluster SYNC; +DROP TABLE IF EXISTS dsolovianov.core_dim_user ON CLUSTER clickhouse_cluster SYNC; +DROP TABLE IF EXISTS dsolovianov.core_dim_course ON CLUSTER clickhouse_cluster SYNC; +DROP TABLE IF EXISTS dsolovianov.core_dim_lesson ON CLUSTER clickhouse_cluster SYNC; +DROP TABLE IF EXISTS dsolovianov.core_dim_date ON CLUSTER clickhouse_cluster SYNC; +DROP TABLE IF EXISTS dsolovianov.core_enrollments ON CLUSTER clickhouse_cluster SYNC; + +CREATE TABLE dsolovianov.core_dim_user +ON CLUSTER clickhouse_cluster +( + user_id Int32, + name String, + age Int32, + email String, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/core_dim_user', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE dsolovianov.core_dim_course +ON CLUSTER clickhouse_cluster +( + course_id Int32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/core_dim_course', + '{replica}' +) +ORDER BY course_id; + +CREATE TABLE dsolovianov.core_dim_lesson +ON CLUSTER clickhouse_cluster +( + lesson_id Int32, + title String, + duration_min Int32, + course_id Int32 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/core_dim_lesson', + '{replica}' +) +ORDER BY lesson_id; + +CREATE TABLE dsolovianov.core_dim_date +ON CLUSTER clickhouse_cluster +( + date Date, + year Int32, + month Int32, + day Int32, + weekday String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/core_dim_date', + '{replica}' +) +ORDER BY date; + +CREATE TABLE dsolovianov.core_enrollments +ON CLUSTER clickhouse_cluster +( + user_id Int32, + course_id Int32, + enrolled_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/core_enrollments', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE dsolovianov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster +( + user_id Int32, + lesson_id Int32, + course_id Int32, + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/core_fact_lesson_views', + '{replica}' +) +ORDER BY user_id; \ No newline at end of file diff --git a/dsolovianov/clickhouse/Pet-Project/Этап 2. Создание витринных таблиц.sql b/dsolovianov/clickhouse/Pet-Project/Этап 2. Создание витринных таблиц.sql new file mode 100644 index 0000000..e2dc606 --- /dev/null +++ b/dsolovianov/clickhouse/Pet-Project/Этап 2. Создание витринных таблиц.sql @@ -0,0 +1,186 @@ +-- Этап 2. Создание витринных таблиц -- + +-- Витрина lesson_popularity_summary +DROP TABLE IF EXISTS dsolovianov.lesson_popularity_summary ON CLUSTER clickhouse_cluster SYNC; + +CREATE TABLE dsolovianov.lesson_popularity_summary ON CLUSTER clickhouse_cluster ( + lesson_id UInt32, + lesson_title String, + course_id UInt32, + course_title String, + total_views UInt64, + unique_users UInt64, + first_view DateTime, + last_view DateTime +) ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/lesson_popularity_summary', + '{replica}' +) +ORDER BY (course_id, lesson_id); + +CREATE MATERIALIZED VIEW dsolovianov.lesson_popularity_summary_mv ON CLUSTER clickhouse_cluster +TO dsolovianov.lesson_popularity_summary +AS +SELECT + l.lesson_id AS lesson_id, + l.title AS lesson_title, + l.course_id AS course_id, + c.title AS course_title, + COUNT(v.viewed_at) AS total_views, + COUNT(DISTINCT v.user_id) AS unique_users, + MIN(v.viewed_at) AS first_view, + MAX(v.viewed_at) AS last_view +FROM dsolovianov.core_fact_lesson_views v +LEFT JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id +LEFT JOIN dsolovianov.core_dim_course c ON l.course_id = c.course_id +GROUP BY l.lesson_id, l.title, l.course_id, c.title; + +INSERT INTO dsolovianov.lesson_popularity_summary +SELECT + l.lesson_id AS lesson_id, + l.title AS lesson_title, + l.course_id AS course_id, + c.title AS course_title, + COUNT(v.viewed_at) AS total_views, + COUNT(DISTINCT v.user_id) AS unique_users, + MIN(v.viewed_at) AS first_view, + MAX(v.viewed_at) AS last_view +FROM dsolovianov.core_dim_lesson l +JOIN dsolovianov.core_dim_course c ON l.course_id = c.course_id +LEFT JOIN dsolovianov.core_fact_lesson_views v ON l.lesson_id = v.lesson_id +GROUP BY l.lesson_id, l.title, l.course_id, c.title; + +-- Витрину inactive_users_summary + +DROP TABLE IF EXISTS dsolovianov.inactive_users_summary ON CLUSTER clickhouse_cluster SYNC; + +CREATE TABLE dsolovianov.inactive_users_summary ON CLUSTER clickhouse_cluster ( + user_id UInt32, + name String, + email String, + age UInt8, + registered_courses_count UInt32, + registration_date Date +) ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/inactive_users_summary', + '{replica}' +) +ORDER BY user_id; + +CREATE MATERIALIZED VIEW dsolovianov.inactive_users_summary_mv ON CLUSTER clickhouse_cluster +TO dsolovianov.inactive_users_summary +AS +SELECT + u.user_id AS user_id, + u.name AS name, + u.email AS email, + u.age AS age, + COUNT(DISTINCT e.course_id) AS registered_courses_count, + u.registration_date AS registration_date +FROM dsolovianov.core_dim_user u +LEFT JOIN dsolovianov.core_enrollments e ON u.user_id = e.user_id +WHERE u.user_id NOT IN (SELECT DISTINCT user_id FROM dsolovianov.core_fact_lesson_views) +GROUP BY u.user_id, u.name, u.email, u.age, u.registration_date; + +INSERT INTO dsolovianov.inactive_users_summary +SELECT + u.user_id AS user_id, + u.name AS name, + u.email AS email, + u.age AS age, + COUNT(DISTINCT e.course_id) AS registered_courses_count, + u.registration_date AS registration_date +FROM dsolovianov.core_dim_user u +LEFT JOIN dsolovianov.core_enrollments e ON u.user_id = e.user_id +WHERE u.user_id NOT IN (SELECT DISTINCT user_id FROM dsolovianov.core_fact_lesson_views) +GROUP BY u.user_id, u.name, u.email, u.age, u.registration_date; + +-- Витрина course_completion_rate + +DROP TABLE IF EXISTS dsolovianov.course_completion_rate ON CLUSTER clickhouse_cluster SYNC; + +CREATE TABLE dsolovianov.course_completion_rate ON CLUSTER clickhouse_cluster ( + user_id UInt32, + user_name String, + course_id UInt32, + course_title String, + lessons_in_course UInt32, + lessons_viewed UInt32, + completion_rate Float32 +) ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/dsolovianov/course_completion_rate', + '{replica}' +) +ORDER BY (user_id, course_id); + +DROP TABLE IF EXISTS dsolovianov.course_completion_rate_mv ON CLUSTER clickhouse_cluster SYNC; + +CREATE MATERIALIZED VIEW dsolovianov.course_completion_rate_mv ON CLUSTER clickhouse_cluster +TO dsolovianov.course_completion_rate +AS +SELECT + e.user_id AS user_id, + u.name AS user_name, + e.course_id AS course_id, + c.title AS course_title, + ( + SELECT COUNT(*) + FROM dsolovianov.core_dim_lesson l + WHERE l.course_id = e.course_id + ) AS lessons_in_course, + ( + SELECT COUNT(DISTINCT v.lesson_id) + FROM dsolovianov.core_fact_lesson_views v + JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id + WHERE v.user_id = e.user_id AND l.course_id = e.course_id + ) AS lessons_viewed, + ( + SELECT COUNT(DISTINCT v.lesson_id) + FROM dsolovianov.core_fact_lesson_views v + JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id + WHERE v.user_id = e.user_id AND l.course_id = e.course_id + ) / ( + SELECT COUNT(*) + FROM dsolovianov.core_dim_lesson l + WHERE l.course_id = e.course_id + ) AS completion_rate +FROM dsolovianov.core_enrollments e +JOIN dsolovianov.core_dim_user u ON e.user_id = u.user_id +JOIN dsolovianov.core_dim_course c ON e.course_id = c.course_id +GROUP BY e.user_id, u.name, e.course_id, c.title; + +INSERT INTO dsolovianov.course_completion_rate +SELECT + e.user_id AS user_id, + u.name AS user_name, + e.course_id AS course_id, + c.title AS course_title, + ( + SELECT COUNT(*) + FROM dsolovianov.core_dim_lesson l + WHERE l.course_id = e.course_id + ) AS lessons_in_course, + COALESCE( + ( + SELECT COUNT(DISTINCT v.lesson_id) + FROM dsolovianov.core_fact_lesson_views v + JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id + WHERE v.user_id = e.user_id AND l.course_id = e.course_id + ), 0 + ) AS lessons_viewed, + COALESCE( + ( + SELECT COUNT(DISTINCT v.lesson_id) + FROM dsolovianov.core_fact_lesson_views v + JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id + WHERE v.user_id = e.user_id AND l.course_id = e.course_id + ), 0 + ) / ( + SELECT COUNT(*) + FROM dsolovianov.core_dim_lesson l + WHERE l.course_id = e.course_id + ) AS completion_rate +FROM dsolovianov.core_enrollments e +JOIN dsolovianov.core_dim_user u ON e.user_id = u.user_id +JOIN dsolovianov.core_dim_course c ON e.course_id = c.course_id +GROUP BY e.user_id, u.name, e.course_id, c.title; diff --git a/dsolovianov/clickhouse/Pet-Project/Этап 3. Анализ и запросы.sql b/dsolovianov/clickhouse/Pet-Project/Этап 3. Анализ и запросы.sql new file mode 100644 index 0000000..6550e5b --- /dev/null +++ b/dsolovianov/clickhouse/Pet-Project/Этап 3. Анализ и запросы.sql @@ -0,0 +1,88 @@ +-- Этап 3. Анализ и запросы - Напиши SQL-запросы к витринам -- Напиши свои вариации для практики-- +-- Топ-5 самых просматриваемых уроков + +SELECT + lesson_title, + course_title, + total_views +FROM lesson_popularity_summary +ORDER BY total_views DESC +LIMIT 5; + +-- Неактивные пользователи, записавшиеся на курсы + +SELECT + name, + email, + registered_courses_count +FROM inactive_users_summary +WHERE registered_courses_count > 0 +ORDER BY registration_date DESC; + +-- Курсы с самым высоким процентом завершения + +SELECT + course_title, + AVG(completion_rate) AS avg_completion +FROM course_completion_rate +GROUP BY course_title +ORDER BY avg_completion DESC +LIMIT 5; + +-- Мои вариации -- + +-- Среднее количество учеников на урок по каждому курсу + +SELECT + course_title, + ROUND(AVG(unique_users), 2) AS avg_users_lesson, + SUM(total_views) AS total_course_views +FROM dsolovianov.lesson_popularity_summary +GROUP BY course_title +ORDER BY avg_users_lesson DESC; + +-- Возрастное распределение неактивных пользователей + +SELECT + CASE + WHEN age < 25 THEN 'до 25' + WHEN age BETWEEN 25 AND 30 THEN '25-30' + WHEN age BETWEEN 31 AND 35 THEN '31-35' + ELSE '35+' + END AS age_group, + COUNT(*) AS inactive_users_count, + AVG(registered_courses_count) AS avg_courses_enrolled +FROM dsolovianov.inactive_users_summary +GROUP BY age_group +ORDER BY age_group; + +-- Пользователи, которые начали, но не закончили курс (completion_rate между 0 и 1) + +SELECT + user_name, + course_title, + lessons_viewed, + lessons_in_course, + ROUND(completion_rate * 100, 0) AS no_completion, + (lessons_in_course - lessons_viewed) AS remaining_lessons +FROM dsolovianov.course_completion_rate +WHERE completion_rate BETWEEN 0.01 AND 0.99 +ORDER BY completion_rate DESC, user_name; + +-- Прогресс каждого пользователя по курсам (с указанием статуса) + +SELECT + user_name, + course_title, + lessons_viewed, + lessons_in_course, + ROUND(completion_rate * 100, 0) AS progress_percent, + CASE + WHEN completion_rate = 1.0 THEN 'Пройден' + WHEN completion_rate = 0 THEN 'Не начат' + WHEN completion_rate >= 0.5 THEN 'В процессе (>50%)' + ELSE 'В процессе (<50%)' + END AS status +FROM dsolovianov.course_completion_rate +ORDER BY user_name, completion_rate DESC; + diff --git a/dsolovianov/clickhouse/Pet-Project/Этап 4. Тестирование и валидация.sql b/dsolovianov/clickhouse/Pet-Project/Этап 4. Тестирование и валидация.sql new file mode 100644 index 0000000..f053160 --- /dev/null +++ b/dsolovianov/clickhouse/Pet-Project/Этап 4. Тестирование и валидация.sql @@ -0,0 +1,13 @@ +-- Этап 4. Тестирование и валидация -- + +/* +### **Задание 6: Сравни данные с Greenplum** + +1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum +2. Сравни с ClickHouse-результатами: + - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX` + - Проверяй `NULL`, `0`, округления +3. Напиши мини-отчёт: **"где совпадает, где нет, почему"** +*/ + +-- Ничего не понятно по этому этапу. У меня в GP нет таблиц "_summary_source"