Pet project is over

itikhomirov/clickhouse/Pet_project.sql

@@ -1,69 +1,92 @@
-CREATE database pet;
-
 -- Этап 1
 -- Задание 1
 -- Перенести данные как в пет-проекте с Greenplum
 
--- Создаем таблицы, чтобы потом загрузить в них данные из таблиц GreenPlum
-
--- Таблица user из GP
-
-CREATE TABLE pet.core_user (
+CREATE TABLE itikhomirov.core_user
+ON CLUSTER clickhouse_cluster
+(
     user_id Int32,
     name String,
     age Int32,
     email String,
     registration_date DateTime64(3, 'UTC')
 )
-ENGINE = MergeTree
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_user',
+    '{replica}'
+)
 ORDER BY user_id;
 
--- Таблица course из GP
-CREATE TABLE pet.core_course (
-    course_id Int32,
-    title String,
-    category String
-)
-ENGINE = MergeTree
-ORDER BY course_id;
+ALTER TABLE itikhomirov.core_user
+MODIFY COLUMN registration_date DateTime64(3);
 
--- Таблица lesson из GP
-CREATE TABLE pet.core_lesson (
+CREATE TABLE itikhomirov.core_lesson 
+ON CLUSTER clickhouse_cluster
+(
     lesson_id Int32,
     title String,
     duration_min Int32,
     course_id Int32
 )
-ENGINE = MergeTree
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_lesson',
+    '{replica}'
+)
 ORDER BY lesson_id;
 
--- Таблица fact_lesson_views из GP
-
-CREATE TABLE pet.core_fact_lesson_views (
-    user_id Int32, 
+CREATE TABLE itikhomirov.core_fact_lesson_views
+ON CLUSTER clickhouse_cluster
+(
+	user_id Int32, 
     lesson_id Int32,
     course_id Int32,
-    viewed_at DateTime64(3, 'UTC')
+    viewed_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views',
+    '{replica}'
 )
-ENGINE = MergeTree
 ORDER BY user_id;
 
--- Таблица enrollments из GP
+ALTER TABLE itikhomirov.core_fact_lesson_views
+MODIFY COLUMN viewed_at DateTime64(3);
 
-CREATE TABLE pet.core_enrollments (
-    user_id Int32,
+DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views
+ON CLUSTER clickhouse_cluster;
+
+CREATE TABLE itikhomirov.core_enrollments
+ON CLUSTER clickhouse_cluster
+(
+	user_id Int32, 
     course_id Int32,
-    enrolled_at DateTime64(3, 'UTC')
+    enrolled_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы',
+    '{replica}'
 )
-ENGINE = MergeTree
 ORDER BY user_id;
 
+CREATE TABLE itikhomirov.core_course
+ON CLUSTER clickhouse_cluster
+(
+    course_id Int32,
+    title String,
+    category String
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_course',
+    '{replica}'
+)
+ORDER BY course_id;
+
 -- Этап 2. Создание витринных таблиц
 -- Задание 2: Подготовь структуру витрины lesson_popularity_summary
 
 -- Создаем пустую таблицу lesson_popularity_summary(структуру витрины)
 
-CREATE TABLE pet.lesson_popularity_summary
+CREATE TABLE itikhomirov.lesson_popularity_summary
+ON CLUSTER clickhouse_cluster
 (
     lesson_id     UInt32,
     lesson_title  String,
@@ -71,16 +94,23 @@ CREATE TABLE pet.lesson_popularity_summary
     course_title  String,
     total_views   UInt32,
     unique_users  UInt32,
-    first_view    DateTime64(3, 'UTC'),
-    last_view     DateTime64(3, 'UTC')
+    first_view    DateTime64(3),
+    last_view     DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary',
+    '{replica}'
 )
-ENGINE = MergeTree
 ORDER BY (lesson_id, course_id);
 
 -- Создаем Materialized View
 
-CREATE MATERIALIZED VIEW pet.mv_lesson_popularity_summary
-TO pet.lesson_popularity_summary
+DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.lesson_popularity_summary
 AS
 SELECT
     flv.lesson_id AS lesson_id,
@@ -91,18 +121,18 @@ SELECT
     uniqExact(flv.user_id) AS unique_users,
     min(flv.viewed_at) AS first_view,
     max(flv.viewed_at) AS last_view
-FROM pet.core_fact_lesson_views AS flv
-JOIN pet.core_lesson  AS l ON flv.lesson_id = l.lesson_id
-JOIN pet.core_course  AS c ON flv.course_id = c.course_id
+FROM itikhomirov.core_fact_lesson_views AS flv
+JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
+JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
 GROUP BY
     flv.lesson_id,
     l.title,
     flv.course_id,
     c.title;
 
--- Вставляем данные в таблицу pet.lesson_popularity_summary
+-- Вставляем данные в таблицу lesson_popularity_summary
 
-INSERT INTO pet.lesson_popularity_summary
+INSERT INTO itikhomirov.lesson_popularity_summary
 SELECT
     flv.lesson_id,
     l.title AS lesson_title,
@@ -112,9 +142,9 @@ SELECT
     uniqExact(flv.user_id) AS unique_users,
     min(flv.viewed_at) AS first_view,
     max(flv.viewed_at) AS last_view
-FROM pet.core_fact_lesson_views AS flv
-JOIN pet.core_lesson  AS l ON flv.lesson_id = l.lesson_id
-JOIN pet.core_course  AS c ON flv.course_id = c.course_id
+FROM itikhomirov.core_fact_lesson_views AS flv
+JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
+JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
 GROUP BY
     flv.lesson_id,
     l.title,
@@ -125,75 +155,93 @@ GROUP BY
 
 -- Создаем пустую таблицу inactive_users_summary(структуру витрины)
 
-CREATE TABLE pet.inactive_users_summary
+DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE TABLE itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster
 (
     user_id UInt32,
     name String,
     email String,
     age UInt8,
     registered_courses_count UInt32,
-    registration_date DateTime64(3, 'UTC')
+    registration_date DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary',
+    '{replica}'
 )
-ENGINE = MergeTree
 ORDER BY (user_id);
 
 -- Создаем Materialized View
 
-CREATE MATERIALIZED VIEW pet.mv_inactive_users_summary
-TO pet.inactive_users_summary
+DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.inactive_users_summary
 AS
+WITH user_courses AS (
+    SELECT
+        e.user_id,
+        countDistinct(e.course_id) AS registered_courses_count
+    FROM itikhomirov.core_enrollments AS e
+    GROUP BY
+        e.user_id
+)
+SELECT
+    u.user_id                                AS user_id,
+    u.name                                   AS name,
+    u.email                                  AS email,
+    u.age                                    AS age,
+    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
+    u.registration_date                      AS registration_date
+FROM itikhomirov.core_user AS u
+LEFT JOIN user_courses AS uc
+    ON u.user_id = uc.user_id
+WHERE u.user_id NOT IN (
+    SELECT DISTINCT user_id
+    FROM itikhomirov.core_fact_lesson_views
+);
+
+
+INSERT INTO itikhomirov.inactive_users_summary
+WITH user_courses AS (
+    SELECT
+        e.user_id,
+        countDistinct(e.course_id) AS registered_courses_count
+    FROM itikhomirov.core_enrollments AS e
+    GROUP BY
+        e.user_id
+)
+-- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров
+-- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views), 
+-- и к ним подмешивается уже готовый registered_courses_count.
 SELECT
     u.user_id AS user_id,
     u.name AS name,
     u.email AS email,
     u.age AS age,
-    countDistinct(e.course_id) AS registered_courses_count,
+    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
     u.registration_date AS registration_date
-FROM pet.core_user AS u
-LEFT JOIN pet.core_enrollments AS e ON u.user_id = e.user_id
+FROM itikhomirov.core_user AS u
+LEFT JOIN user_courses AS uc
+    ON u.user_id = uc.user_id
 WHERE u.user_id NOT IN (
     SELECT DISTINCT user_id
-    FROM pet.core_fact_lesson_views
+    FROM itikhomirov.core_fact_lesson_views
 )
-GROUP BY
-    u.user_id,
-    u.name,
-    u.email,
-    u.age,
-    u.registration_date;
-
--- Вставляем данные в таблицу inactive_users_summary
-
-INSERT INTO pet.inactive_users_summary
-SELECT
-    u.user_id,
-    u.name,
-    u.email,
-    u.age,
-    countDistinct(e.course_id) AS registered_courses_count,
-    u.registration_date
-FROM pet.core_user AS u
-LEFT JOIN pet.core_enrollments AS e
-    ON u.user_id = e.user_id
-WHERE u.user_id NOT IN (               
-    SELECT DISTINCT user_id
-    FROM pet.core_fact_lesson_views
-)
-/*
- * этот подзапрос WHERE... просто выбрасывает всех, кто хоть раз что‑то посмотрел,
- * независимо от количества строк и особенностей JOIN’ов.
- */
-GROUP BY
-    u.user_id,
-    u.name,
-    u.email,
-    u.age,
-    u.registration_date;
-
+ORDER BY u.user_id;
 
 -- Задание 4: Создай витрину course_completion_rate
 
-CREATE TABLE pet.course_completion_rate
+CREATE TABLE itikhomirov.course_completion_rate
+ON CLUSTER clickhouse_cluster
 (
     user_id          UInt32,
     user_name        String,
@@ -203,65 +251,41 @@ CREATE TABLE pet.course_completion_rate
     lessons_viewed    UInt32,
     completion_rate   Float32
 )
-ENGINE = MergeTree
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/course_completion_rate',
+    '{replica}'
+)
 ORDER BY (user_id, course_id);
 
+
 -- Создаем Materialized View
 
-CREATE MATERIALIZED VIEW pet.mv_course_completion_rate
-TO pet.course_completion_rate
+CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.course_completion_rate
 AS
-SELECT
-    u.user_id AS user_id,
-    u.name AS user_name,
-    c.course_id AS course_id,
-    c.title AS course_title,
-    countDistinct(l.lesson_id) AS lessons_in_course,
-    countDistinct(flv.lesson_id) AS lessons_viewed,
-    if(countDistinct(l.lesson_id) = 0,0,
-       round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id),2)
-    ) AS completion_rate
-FROM pet.core_enrollments AS e
-JOIN pet.core_user      AS u ON e.user_id   = u.user_id
-JOIN pet.core_course    AS c ON e.course_id = c.course_id
-JOIN pet.core_lesson    AS l ON c.course_id = l.course_id
-LEFT JOIN pet.core_fact_lesson_views AS flv
-       ON flv.user_id   = e.user_id
-      AND flv.course_id = e.course_id
-GROUP BY
-    u.user_id,
-    u.name,
-    c.course_id,
-    c.title;
-
--- Вставляем данные в таблицу pet.course_completion_rate
-INSERT INTO pet.course_completion_rate
--- Создаем две временные таблицы:
--- 1) Сколько уроков в каждом курсе
 WITH
 lessons_per_course AS (
     SELECT
         course_id,
         countDistinct(lesson_id) AS lessons_in_course
-    FROM pet.core_lesson
+    FROM itikhomirov.core_lesson
     GROUP BY course_id
 ),
--- 2) Сколько уроков каждого курса посмотрел пользователь
 views_per_user_course AS (
     SELECT
         user_id,
         course_id,
         countDistinct(lesson_id) AS lessons_viewed
-    FROM pet.core_fact_lesson_views
+    FROM itikhomirov.core_fact_lesson_views
     GROUP BY user_id, course_id
 )
--- Основной запрос
 SELECT
-    u.user_id,
+    u.user_id AS user_id,
     u.name AS user_name,
-    c.course_id,
+    c.course_id AS course_id,
     c.title AS course_title,
-    lpc.lessons_in_course,
+    lpc.lessons_in_course AS lessons_in_course,
     coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
     ROUND(
         COALESCE(
@@ -271,12 +295,124 @@ SELECT
         ),
         2
     ) AS completion_rate
-FROM pet.core_user AS u
-JOIN pet.core_enrollments AS e ON u.user_id  = e.user_id
-JOIN pet.core_course AS c ON e.course_id = c.course_id
+FROM itikhomirov.core_user AS u
+JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
+JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
 JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
 LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
 ORDER BY
     u.user_id,
     c.course_id;
 
+-- Вставляем данные в таблицу pet.course_completion_rate
+
+INSERT INTO itikhomirov.course_completion_rate
+-- Создаем две временные таблицы:
+-- 1) Сколько уроков в каждом курсе
+WITH
+lessons_per_course AS (
+    SELECT
+        course_id,
+        countDistinct(lesson_id) AS lessons_in_course
+    FROM itikhomirov.core_lesson
+    GROUP BY course_id
+),
+-- 2) Сколько уроков каждого курса посмотрел пользователь
+views_per_user_course AS (
+    SELECT
+        user_id,
+        course_id,
+        countDistinct(lesson_id) AS lessons_viewed
+    FROM itikhomirov.core_fact_lesson_views
+    GROUP BY user_id, course_id
+)
+-- Основной запрос
+SELECT
+    u.user_id AS user_id, 
+    u.name AS user_name,
+    c.course_id AS course_id,
+    c.title AS course_title,
+    lpc.lessons_in_course AS lessons_in_course,
+    coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
+    ROUND(
+        COALESCE(
+            NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
+            / NULLIF(lpc.lessons_in_course * 1.0, 0),
+            0
+        ),
+        2
+    ) AS completion_rate
+FROM itikhomirov.core_user AS u
+JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
+JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
+JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
+LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
+ORDER BY
+    u.user_id,
+    c.course_id;
+
+-- Этап 3. Анализ и запросы
+
+-- Задание 5: Напиши SQL-запросы к витринам
+
+-- Уроки с хотя бы одним просмотром
+SELECT
+    lesson_title,
+    course_title,
+    total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE total_views > 0
+ORDER BY lesson_title ASC;
+
+-- Топ‑3 урока внутри одного курса
+SELECT
+    lesson_title,
+    course_title,
+    total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE course_title = 'SQL для начинающих'
+ORDER BY total_views DESC
+LIMIT 3;
+
+-- Пользователи, которые завершили курсы на 100%
+SELECT
+    user_id,
+    user_name,
+    course_title,
+    completion_rate
+FROM itikhomirov.course_completion_rate
+WHERE completion_rate = 1
+ORDER BY user_id;
+
+-- Этап 4. Тестирование и валидация
+
+-- Задание 6: Сравни данные с Greenplum**
+
+-- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
+
+
+-- 2. Сравни с ClickHouse-результатами:
+--    - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
+--    - Проверяй `NULL`, `0`, округления
+
+
+-- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
+
+SELECT
+    sum(total_views)                 AS sum_total_views,
+    avg(total_views)                 AS avg_total_views,
+    min(total_views)                 AS min_total_views,
+    max(total_views)                 AS max_total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE lesson_id IN (1, 3, 5);
+
+SELECT
+    sum(total_views)                 AS sum_total_views,
+    avg(total_views)                 AS avg_total_views,
+    min(total_views)                 AS min_total_views,
+    max(total_views)                 AS max_total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE lesson_id = 999;
+
+-- Отчёт в .doc файле
+