diff --git a/itikhomirov/.DS_Store b/itikhomirov/.DS_Store index 5a7e020..0215a06 100644 Binary files a/itikhomirov/.DS_Store and b/itikhomirov/.DS_Store differ diff --git a/itikhomirov/clickhouse/Pet_project.sql b/itikhomirov/clickhouse/Pet_project.sql index 2f2f1e7..27c2967 100644 --- a/itikhomirov/clickhouse/Pet_project.sql +++ b/itikhomirov/clickhouse/Pet_project.sql @@ -1,69 +1,92 @@ -CREATE database pet; - -- Этап 1 -- Задание 1 -- Перенести данные как в пет-проекте с Greenplum --- Создаем таблицы, чтобы потом загрузить в них данные из таблиц GreenPlum - --- Таблица user из GP - -CREATE TABLE pet.core_user ( +CREATE TABLE itikhomirov.core_user +ON CLUSTER clickhouse_cluster +( user_id Int32, name String, age Int32, email String, registration_date DateTime64(3, 'UTC') ) -ENGINE = MergeTree +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_user', + '{replica}' +) ORDER BY user_id; --- Таблица course из GP -CREATE TABLE pet.core_course ( - course_id Int32, - title String, - category String -) -ENGINE = MergeTree -ORDER BY course_id; +ALTER TABLE itikhomirov.core_user +MODIFY COLUMN registration_date DateTime64(3); --- Таблица lesson из GP -CREATE TABLE pet.core_lesson ( +CREATE TABLE itikhomirov.core_lesson +ON CLUSTER clickhouse_cluster +( lesson_id Int32, title String, duration_min Int32, course_id Int32 ) -ENGINE = MergeTree +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_lesson', + '{replica}' +) ORDER BY lesson_id; --- Таблица fact_lesson_views из GP - -CREATE TABLE pet.core_fact_lesson_views ( - user_id Int32, +CREATE TABLE itikhomirov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster +( + user_id Int32, lesson_id Int32, course_id Int32, - viewed_at DateTime64(3, 'UTC') + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views', + '{replica}' ) -ENGINE = MergeTree ORDER BY user_id; --- Таблица enrollments из GP +ALTER TABLE itikhomirov.core_fact_lesson_views +MODIFY COLUMN viewed_at DateTime64(3); -CREATE TABLE pet.core_enrollments ( - user_id Int32, +DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster; + +CREATE TABLE itikhomirov.core_enrollments +ON CLUSTER clickhouse_cluster +( + user_id Int32, course_id Int32, - enrolled_at DateTime64(3, 'UTC') + enrolled_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы', + '{replica}' ) -ENGINE = MergeTree ORDER BY user_id; +CREATE TABLE itikhomirov.core_course +ON CLUSTER clickhouse_cluster +( + course_id Int32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/core_course', + '{replica}' +) +ORDER BY course_id; + -- Этап 2. Создание витринных таблиц -- Задание 2: Подготовь структуру витрины lesson_popularity_summary -- Создаем пустую таблицу lesson_popularity_summary(структуру витрины) -CREATE TABLE pet.lesson_popularity_summary +CREATE TABLE itikhomirov.lesson_popularity_summary +ON CLUSTER clickhouse_cluster ( lesson_id UInt32, lesson_title String, @@ -71,16 +94,23 @@ CREATE TABLE pet.lesson_popularity_summary course_title String, total_views UInt32, unique_users UInt32, - first_view DateTime64(3, 'UTC'), - last_view DateTime64(3, 'UTC') + first_view DateTime64(3), + last_view DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary', + '{replica}' ) -ENGINE = MergeTree ORDER BY (lesson_id, course_id); -- Создаем Materialized View -CREATE MATERIALIZED VIEW pet.mv_lesson_popularity_summary -TO pet.lesson_popularity_summary +DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary +ON CLUSTER clickhouse_cluster; + +CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary +ON CLUSTER clickhouse_cluster +TO itikhomirov.lesson_popularity_summary AS SELECT flv.lesson_id AS lesson_id, @@ -91,18 +121,18 @@ SELECT uniqExact(flv.user_id) AS unique_users, min(flv.viewed_at) AS first_view, max(flv.viewed_at) AS last_view -FROM pet.core_fact_lesson_views AS flv -JOIN pet.core_lesson AS l ON flv.lesson_id = l.lesson_id -JOIN pet.core_course AS c ON flv.course_id = c.course_id +FROM itikhomirov.core_fact_lesson_views AS flv +JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id GROUP BY flv.lesson_id, l.title, flv.course_id, c.title; --- Вставляем данные в таблицу pet.lesson_popularity_summary +-- Вставляем данные в таблицу lesson_popularity_summary -INSERT INTO pet.lesson_popularity_summary +INSERT INTO itikhomirov.lesson_popularity_summary SELECT flv.lesson_id, l.title AS lesson_title, @@ -112,9 +142,9 @@ SELECT uniqExact(flv.user_id) AS unique_users, min(flv.viewed_at) AS first_view, max(flv.viewed_at) AS last_view -FROM pet.core_fact_lesson_views AS flv -JOIN pet.core_lesson AS l ON flv.lesson_id = l.lesson_id -JOIN pet.core_course AS c ON flv.course_id = c.course_id +FROM itikhomirov.core_fact_lesson_views AS flv +JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id GROUP BY flv.lesson_id, l.title, @@ -125,75 +155,93 @@ GROUP BY -- Создаем пустую таблицу inactive_users_summary(структуру витрины) -CREATE TABLE pet.inactive_users_summary +DROP TABLE IF EXISTS itikhomirov.inactive_users_summary +ON CLUSTER clickhouse_cluster; + +DROP TABLE IF EXISTS itikhomirov.inactive_users_summary +ON CLUSTER clickhouse_cluster; + +CREATE TABLE itikhomirov.inactive_users_summary +ON CLUSTER clickhouse_cluster ( user_id UInt32, name String, email String, age UInt8, registered_courses_count UInt32, - registration_date DateTime64(3, 'UTC') + registration_date DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary', + '{replica}' ) -ENGINE = MergeTree ORDER BY (user_id); -- Создаем Materialized View -CREATE MATERIALIZED VIEW pet.mv_inactive_users_summary -TO pet.inactive_users_summary +DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary +ON CLUSTER clickhouse_cluster; + +CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary +ON CLUSTER clickhouse_cluster +TO itikhomirov.inactive_users_summary AS +WITH user_courses AS ( + SELECT + e.user_id, + countDistinct(e.course_id) AS registered_courses_count + FROM itikhomirov.core_enrollments AS e + GROUP BY + e.user_id +) +SELECT + u.user_id AS user_id, + u.name AS name, + u.email AS email, + u.age AS age, + coalesce(uc.registered_courses_count, 0) AS registered_courses_count, + u.registration_date AS registration_date +FROM itikhomirov.core_user AS u +LEFT JOIN user_courses AS uc + ON u.user_id = uc.user_id +WHERE u.user_id NOT IN ( + SELECT DISTINCT user_id + FROM itikhomirov.core_fact_lesson_views +); + + +INSERT INTO itikhomirov.inactive_users_summary +WITH user_courses AS ( + SELECT + e.user_id, + countDistinct(e.course_id) AS registered_courses_count + FROM itikhomirov.core_enrollments AS e + GROUP BY + e.user_id +) +-- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров +-- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views), +-- и к ним подмешивается уже готовый registered_courses_count. SELECT u.user_id AS user_id, u.name AS name, u.email AS email, u.age AS age, - countDistinct(e.course_id) AS registered_courses_count, + coalesce(uc.registered_courses_count, 0) AS registered_courses_count, u.registration_date AS registration_date -FROM pet.core_user AS u -LEFT JOIN pet.core_enrollments AS e ON u.user_id = e.user_id +FROM itikhomirov.core_user AS u +LEFT JOIN user_courses AS uc + ON u.user_id = uc.user_id WHERE u.user_id NOT IN ( SELECT DISTINCT user_id - FROM pet.core_fact_lesson_views + FROM itikhomirov.core_fact_lesson_views ) -GROUP BY - u.user_id, - u.name, - u.email, - u.age, - u.registration_date; - --- Вставляем данные в таблицу inactive_users_summary - -INSERT INTO pet.inactive_users_summary -SELECT - u.user_id, - u.name, - u.email, - u.age, - countDistinct(e.course_id) AS registered_courses_count, - u.registration_date -FROM pet.core_user AS u -LEFT JOIN pet.core_enrollments AS e - ON u.user_id = e.user_id -WHERE u.user_id NOT IN ( - SELECT DISTINCT user_id - FROM pet.core_fact_lesson_views -) -/* - * этот подзапрос WHERE... просто выбрасывает всех, кто хоть раз что‑то посмотрел, - * независимо от количества строк и особенностей JOIN’ов. - */ -GROUP BY - u.user_id, - u.name, - u.email, - u.age, - u.registration_date; - +ORDER BY u.user_id; -- Задание 4: Создай витрину course_completion_rate -CREATE TABLE pet.course_completion_rate +CREATE TABLE itikhomirov.course_completion_rate +ON CLUSTER clickhouse_cluster ( user_id UInt32, user_name String, @@ -203,65 +251,41 @@ CREATE TABLE pet.course_completion_rate lessons_viewed UInt32, completion_rate Float32 ) -ENGINE = MergeTree +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/itikhomirov/course_completion_rate', + '{replica}' +) ORDER BY (user_id, course_id); + -- Создаем Materialized View -CREATE MATERIALIZED VIEW pet.mv_course_completion_rate -TO pet.course_completion_rate +CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate +ON CLUSTER clickhouse_cluster +TO itikhomirov.course_completion_rate AS -SELECT - u.user_id AS user_id, - u.name AS user_name, - c.course_id AS course_id, - c.title AS course_title, - countDistinct(l.lesson_id) AS lessons_in_course, - countDistinct(flv.lesson_id) AS lessons_viewed, - if(countDistinct(l.lesson_id) = 0,0, - round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id),2) - ) AS completion_rate -FROM pet.core_enrollments AS e -JOIN pet.core_user AS u ON e.user_id = u.user_id -JOIN pet.core_course AS c ON e.course_id = c.course_id -JOIN pet.core_lesson AS l ON c.course_id = l.course_id -LEFT JOIN pet.core_fact_lesson_views AS flv - ON flv.user_id = e.user_id - AND flv.course_id = e.course_id -GROUP BY - u.user_id, - u.name, - c.course_id, - c.title; - --- Вставляем данные в таблицу pet.course_completion_rate -INSERT INTO pet.course_completion_rate --- Создаем две временные таблицы: --- 1) Сколько уроков в каждом курсе WITH lessons_per_course AS ( SELECT course_id, countDistinct(lesson_id) AS lessons_in_course - FROM pet.core_lesson + FROM itikhomirov.core_lesson GROUP BY course_id ), --- 2) Сколько уроков каждого курса посмотрел пользователь views_per_user_course AS ( SELECT user_id, course_id, countDistinct(lesson_id) AS lessons_viewed - FROM pet.core_fact_lesson_views + FROM itikhomirov.core_fact_lesson_views GROUP BY user_id, course_id ) --- Основной запрос SELECT - u.user_id, + u.user_id AS user_id, u.name AS user_name, - c.course_id, + c.course_id AS course_id, c.title AS course_title, - lpc.lessons_in_course, + lpc.lessons_in_course AS lessons_in_course, coalesce(vpc.lessons_viewed, 0) AS lessons_viewed, ROUND( COALESCE( @@ -271,12 +295,124 @@ SELECT ), 2 ) AS completion_rate -FROM pet.core_user AS u -JOIN pet.core_enrollments AS e ON u.user_id = e.user_id -JOIN pet.core_course AS c ON e.course_id = c.course_id +FROM itikhomirov.core_user AS u +JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id +JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id ORDER BY u.user_id, c.course_id; +-- Вставляем данные в таблицу pet.course_completion_rate + +INSERT INTO itikhomirov.course_completion_rate +-- Создаем две временные таблицы: +-- 1) Сколько уроков в каждом курсе +WITH +lessons_per_course AS ( + SELECT + course_id, + countDistinct(lesson_id) AS lessons_in_course + FROM itikhomirov.core_lesson + GROUP BY course_id +), +-- 2) Сколько уроков каждого курса посмотрел пользователь +views_per_user_course AS ( + SELECT + user_id, + course_id, + countDistinct(lesson_id) AS lessons_viewed + FROM itikhomirov.core_fact_lesson_views + GROUP BY user_id, course_id +) +-- Основной запрос +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + lpc.lessons_in_course AS lessons_in_course, + coalesce(vpc.lessons_viewed, 0) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0) + / NULLIF(lpc.lessons_in_course * 1.0, 0), + 0 + ), + 2 + ) AS completion_rate +FROM itikhomirov.core_user AS u +JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id +JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id +JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id +LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id +ORDER BY + u.user_id, + c.course_id; + +-- Этап 3. Анализ и запросы + +-- Задание 5: Напиши SQL-запросы к витринам + +-- Уроки с хотя бы одним просмотром +SELECT + lesson_title, + course_title, + total_views +FROM itikhomirov.lesson_popularity_summary +WHERE total_views > 0 +ORDER BY lesson_title ASC; + +-- Топ‑3 урока внутри одного курса +SELECT + lesson_title, + course_title, + total_views +FROM itikhomirov.lesson_popularity_summary +WHERE course_title = 'SQL для начинающих' +ORDER BY total_views DESC +LIMIT 3; + +-- Пользователи, которые завершили курсы на 100% +SELECT + user_id, + user_name, + course_title, + completion_rate +FROM itikhomirov.course_completion_rate +WHERE completion_rate = 1 +ORDER BY user_id; + +-- Этап 4. Тестирование и валидация + +-- Задание 6: Сравни данные с Greenplum** + +-- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum + + +-- 2. Сравни с ClickHouse-результатами: +-- - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX` +-- - Проверяй `NULL`, `0`, округления + + +-- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"** + +SELECT + sum(total_views) AS sum_total_views, + avg(total_views) AS avg_total_views, + min(total_views) AS min_total_views, + max(total_views) AS max_total_views +FROM itikhomirov.lesson_popularity_summary +WHERE lesson_id IN (1, 3, 5); + +SELECT + sum(total_views) AS sum_total_views, + avg(total_views) AS avg_total_views, + min(total_views) AS min_total_views, + max(total_views) AS max_total_views +FROM itikhomirov.lesson_popularity_summary +WHERE lesson_id = 999; + +-- Отчёт в .doc файле + diff --git a/itikhomirov/clickhouse/Pet_project_Step 1.1.png b/itikhomirov/clickhouse/Pet_project_Step 1.1.png new file mode 100644 index 0000000..82fc7ae Binary files /dev/null and b/itikhomirov/clickhouse/Pet_project_Step 1.1.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 1.png b/itikhomirov/clickhouse/Pet_project_Step 1.png deleted file mode 100644 index 9a9bea5..0000000 Binary files a/itikhomirov/clickhouse/Pet_project_Step 1.png and /dev/null differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 2.2.png b/itikhomirov/clickhouse/Pet_project_Step 2.2.png index 7061540..81d64b7 100644 Binary files a/itikhomirov/clickhouse/Pet_project_Step 2.2.png and b/itikhomirov/clickhouse/Pet_project_Step 2.2.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 2.3.png b/itikhomirov/clickhouse/Pet_project_Step 2.3.png index 8054f18..b3cc847 100644 Binary files a/itikhomirov/clickhouse/Pet_project_Step 2.3.png and b/itikhomirov/clickhouse/Pet_project_Step 2.3.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 2.4.png b/itikhomirov/clickhouse/Pet_project_Step 2.4.png index 351cff7..237d974 100644 Binary files a/itikhomirov/clickhouse/Pet_project_Step 2.4.png and b/itikhomirov/clickhouse/Pet_project_Step 2.4.png differ diff --git a/itikhomirov/clickhouse/Pet_project_Step 3.5.png b/itikhomirov/clickhouse/Pet_project_Step 3.5.png new file mode 100644 index 0000000..ed6d534 Binary files /dev/null and b/itikhomirov/clickhouse/Pet_project_Step 3.5.png differ diff --git a/itikhomirov/clickhouse/Step 4.6.pdf b/itikhomirov/clickhouse/Step 4.6.pdf new file mode 100644 index 0000000..804ddc2 Binary files /dev/null and b/itikhomirov/clickhouse/Step 4.6.pdf differ