Merge pull request 'add Step 1 from pet project Clickhouse' (#19) from OS-68 into main

Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/19

itikhomirov/clickhouse/Pet_project.sql

@@ -0,0 +1,418 @@
+-- Этап 1
+-- Задание 1
+-- Перенести данные как в пет-проекте с Greenplum
+
+CREATE TABLE itikhomirov.core_user
+ON CLUSTER clickhouse_cluster
+(
+    user_id Int32,
+    name String,
+    age Int32,
+    email String,
+    registration_date DateTime64(3, 'UTC')
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_user',
+    '{replica}'
+)
+ORDER BY user_id;
+
+ALTER TABLE itikhomirov.core_user
+MODIFY COLUMN registration_date DateTime64(3);
+
+CREATE TABLE itikhomirov.core_lesson 
+ON CLUSTER clickhouse_cluster
+(
+    lesson_id Int32,
+    title String,
+    duration_min Int32,
+    course_id Int32
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_lesson',
+    '{replica}'
+)
+ORDER BY lesson_id;
+
+CREATE TABLE itikhomirov.core_fact_lesson_views
+ON CLUSTER clickhouse_cluster
+(
+	user_id Int32, 
+    lesson_id Int32,
+    course_id Int32,
+    viewed_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views',
+    '{replica}'
+)
+ORDER BY user_id;
+
+ALTER TABLE itikhomirov.core_fact_lesson_views
+MODIFY COLUMN viewed_at DateTime64(3);
+
+DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views
+ON CLUSTER clickhouse_cluster;
+
+CREATE TABLE itikhomirov.core_enrollments
+ON CLUSTER clickhouse_cluster
+(
+	user_id Int32, 
+    course_id Int32,
+    enrolled_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы',
+    '{replica}'
+)
+ORDER BY user_id;
+
+CREATE TABLE itikhomirov.core_course
+ON CLUSTER clickhouse_cluster
+(
+    course_id Int32,
+    title String,
+    category String
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_course',
+    '{replica}'
+)
+ORDER BY course_id;
+
+-- Этап 2. Создание витринных таблиц
+-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
+
+-- Создаем пустую таблицу lesson_popularity_summary(структуру витрины)
+
+CREATE TABLE itikhomirov.lesson_popularity_summary
+ON CLUSTER clickhouse_cluster
+(
+    lesson_id     UInt32,
+    lesson_title  String,
+    course_id     UInt32,
+    course_title  String,
+    total_views   UInt32,
+    unique_users  UInt32,
+    first_view    DateTime64(3),
+    last_view     DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary',
+    '{replica}'
+)
+ORDER BY (lesson_id, course_id);
+
+-- Создаем Materialized View
+
+DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.lesson_popularity_summary
+AS
+SELECT
+    flv.lesson_id AS lesson_id,
+    l.title AS lesson_title,
+    flv.course_id AS course_id,
+    c.title AS course_title,
+    count() AS total_views,
+    uniqExact(flv.user_id) AS unique_users,
+    min(flv.viewed_at) AS first_view,
+    max(flv.viewed_at) AS last_view
+FROM itikhomirov.core_fact_lesson_views AS flv
+JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
+JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
+GROUP BY
+    flv.lesson_id,
+    l.title,
+    flv.course_id,
+    c.title;
+
+-- Вставляем данные в таблицу lesson_popularity_summary
+
+INSERT INTO itikhomirov.lesson_popularity_summary
+SELECT
+    flv.lesson_id,
+    l.title AS lesson_title,
+    flv.course_id,
+    c.title AS course_title,
+    count() AS total_views,
+    uniqExact(flv.user_id) AS unique_users,
+    min(flv.viewed_at) AS first_view,
+    max(flv.viewed_at) AS last_view
+FROM itikhomirov.core_fact_lesson_views AS flv
+JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
+JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
+GROUP BY
+    flv.lesson_id,
+    l.title,
+    flv.course_id,
+    c.title;
+
+-- Задание 3: Создай витрину inactive_users_summary
+
+-- Создаем пустую таблицу inactive_users_summary(структуру витрины)
+
+DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE TABLE itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster
+(
+    user_id UInt32,
+    name String,
+    email String,
+    age UInt8,
+    registered_courses_count UInt32,
+    registration_date DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary',
+    '{replica}'
+)
+ORDER BY (user_id);
+
+-- Создаем Materialized View
+
+DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.inactive_users_summary
+AS
+WITH user_courses AS (
+    SELECT
+        e.user_id,
+        countDistinct(e.course_id) AS registered_courses_count
+    FROM itikhomirov.core_enrollments AS e
+    GROUP BY
+        e.user_id
+)
+SELECT
+    u.user_id                                AS user_id,
+    u.name                                   AS name,
+    u.email                                  AS email,
+    u.age                                    AS age,
+    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
+    u.registration_date                      AS registration_date
+FROM itikhomirov.core_user AS u
+LEFT JOIN user_courses AS uc
+    ON u.user_id = uc.user_id
+WHERE u.user_id NOT IN (
+    SELECT DISTINCT user_id
+    FROM itikhomirov.core_fact_lesson_views
+);
+
+
+INSERT INTO itikhomirov.inactive_users_summary
+WITH user_courses AS (
+    SELECT
+        e.user_id,
+        countDistinct(e.course_id) AS registered_courses_count
+    FROM itikhomirov.core_enrollments AS e
+    GROUP BY
+        e.user_id
+)
+-- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров
+-- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views), 
+-- и к ним подмешивается уже готовый registered_courses_count.
+SELECT
+    u.user_id AS user_id,
+    u.name AS name,
+    u.email AS email,
+    u.age AS age,
+    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
+    u.registration_date AS registration_date
+FROM itikhomirov.core_user AS u
+LEFT JOIN user_courses AS uc
+    ON u.user_id = uc.user_id
+WHERE u.user_id NOT IN (
+    SELECT DISTINCT user_id
+    FROM itikhomirov.core_fact_lesson_views
+)
+ORDER BY u.user_id;
+
+-- Задание 4: Создай витрину course_completion_rate
+
+CREATE TABLE itikhomirov.course_completion_rate
+ON CLUSTER clickhouse_cluster
+(
+    user_id          UInt32,
+    user_name        String,
+    course_id        UInt32,
+    course_title     String,
+    lessons_in_course UInt32,
+    lessons_viewed    UInt32,
+    completion_rate   Float32
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/course_completion_rate',
+    '{replica}'
+)
+ORDER BY (user_id, course_id);
+
+
+-- Создаем Materialized View
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.course_completion_rate
+AS
+WITH
+lessons_per_course AS (
+    SELECT
+        course_id,
+        countDistinct(lesson_id) AS lessons_in_course
+    FROM itikhomirov.core_lesson
+    GROUP BY course_id
+),
+views_per_user_course AS (
+    SELECT
+        user_id,
+        course_id,
+        countDistinct(lesson_id) AS lessons_viewed
+    FROM itikhomirov.core_fact_lesson_views
+    GROUP BY user_id, course_id
+)
+SELECT
+    u.user_id AS user_id,
+    u.name AS user_name,
+    c.course_id AS course_id,
+    c.title AS course_title,
+    lpc.lessons_in_course AS lessons_in_course,
+    coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
+    ROUND(
+        COALESCE(
+            NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
+            / NULLIF(lpc.lessons_in_course * 1.0, 0),
+            0
+        ),
+        2
+    ) AS completion_rate
+FROM itikhomirov.core_user AS u
+JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
+JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
+JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
+LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
+ORDER BY
+    u.user_id,
+    c.course_id;
+
+-- Вставляем данные в таблицу pet.course_completion_rate
+
+INSERT INTO itikhomirov.course_completion_rate
+-- Создаем две временные таблицы:
+-- 1) Сколько уроков в каждом курсе
+WITH
+lessons_per_course AS (
+    SELECT
+        course_id,
+        countDistinct(lesson_id) AS lessons_in_course
+    FROM itikhomirov.core_lesson
+    GROUP BY course_id
+),
+-- 2) Сколько уроков каждого курса посмотрел пользователь
+views_per_user_course AS (
+    SELECT
+        user_id,
+        course_id,
+        countDistinct(lesson_id) AS lessons_viewed
+    FROM itikhomirov.core_fact_lesson_views
+    GROUP BY user_id, course_id
+)
+-- Основной запрос
+SELECT
+    u.user_id AS user_id, 
+    u.name AS user_name,
+    c.course_id AS course_id,
+    c.title AS course_title,
+    lpc.lessons_in_course AS lessons_in_course,
+    coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
+    ROUND(
+        COALESCE(
+            NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
+            / NULLIF(lpc.lessons_in_course * 1.0, 0),
+            0
+        ),
+        2
+    ) AS completion_rate
+FROM itikhomirov.core_user AS u
+JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
+JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
+JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
+LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
+ORDER BY
+    u.user_id,
+    c.course_id;
+
+-- Этап 3. Анализ и запросы
+
+-- Задание 5: Напиши SQL-запросы к витринам
+
+-- Уроки с хотя бы одним просмотром
+SELECT
+    lesson_title,
+    course_title,
+    total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE total_views > 0
+ORDER BY lesson_title ASC;
+
+-- Топ‑3 урока внутри одного курса
+SELECT
+    lesson_title,
+    course_title,
+    total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE course_title = 'SQL для начинающих'
+ORDER BY total_views DESC
+LIMIT 3;
+
+-- Пользователи, которые завершили курсы на 100%
+SELECT
+    user_id,
+    user_name,
+    course_title,
+    completion_rate
+FROM itikhomirov.course_completion_rate
+WHERE completion_rate = 1
+ORDER BY user_id;
+
+-- Этап 4. Тестирование и валидация
+
+-- Задание 6: Сравни данные с Greenplum**
+
+-- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
+
+
+-- 2. Сравни с ClickHouse-результатами:
+--    - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
+--    - Проверяй `NULL`, `0`, округления
+
+
+-- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
+
+SELECT
+    sum(total_views)                 AS sum_total_views,
+    avg(total_views)                 AS avg_total_views,
+    min(total_views)                 AS min_total_views,
+    max(total_views)                 AS max_total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE lesson_id IN (1, 3, 5);
+
+SELECT
+    sum(total_views)                 AS sum_total_views,
+    avg(total_views)                 AS avg_total_views,
+    min(total_views)                 AS min_total_views,
+    max(total_views)                 AS max_total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE lesson_id = 999;
+
+-- Отчёт в .doc файле
+