Merge pull request 'add Step 1 from pet project Clickhouse' (#19) from OS-68 into main

Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/19
2025-12-13 13:02:53 +04:00
parent 53398bc3ad 9cbf71a160
commit 8283ecc789
7 changed files with 418 additions and 0 deletions
@@ -0,0 +1,418 @@
 -- Этап 1
 -- Задание 1
 -- Перенести данные как в пет-проекте с Greenplum
 CREATE TABLE itikhomirov.core_user
 ON CLUSTER clickhouse_cluster
 (
    user_id Int32,
    name String,
    age Int32,
    email String,
    registration_date DateTime64(3, 'UTC')
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/core_user',
    '{replica}'
 )
 ORDER BY user_id;
 ALTER TABLE itikhomirov.core_user
 MODIFY COLUMN registration_date DateTime64(3);
 CREATE TABLE itikhomirov.core_lesson 
 ON CLUSTER clickhouse_cluster
 (
    lesson_id Int32,
    title String,
    duration_min Int32,
    course_id Int32
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/core_lesson',
    '{replica}'
 )
 ORDER BY lesson_id;
 CREATE TABLE itikhomirov.core_fact_lesson_views
 ON CLUSTER clickhouse_cluster
 (
 	user_id Int32, 
    lesson_id Int32,
    course_id Int32,
    viewed_at DateTime64(3)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views',
    '{replica}'
 )
 ORDER BY user_id;
 ALTER TABLE itikhomirov.core_fact_lesson_views
 MODIFY COLUMN viewed_at DateTime64(3);
 DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views
 ON CLUSTER clickhouse_cluster;
 CREATE TABLE itikhomirov.core_enrollments
 ON CLUSTER clickhouse_cluster
 (
 	user_id Int32, 
    course_id Int32,
    enrolled_at DateTime64(3)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE TABLE itikhomirov.core_course
 ON CLUSTER clickhouse_cluster
 (
    course_id Int32,
    title String,
    category String
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/core_course',
    '{replica}'
 )
 ORDER BY course_id;
 -- Этап 2. Создание витринных таблиц
 -- Задание 2: Подготовь структуру витрины lesson_popularity_summary
 -- Создаем пустую таблицу lesson_popularity_summary(структуру витрины)
 CREATE TABLE itikhomirov.lesson_popularity_summary
 ON CLUSTER clickhouse_cluster
 (
    lesson_id     UInt32,
    lesson_title  String,
    course_id     UInt32,
    course_title  String,
    total_views   UInt32,
    unique_users  UInt32,
    first_view    DateTime64(3),
    last_view     DateTime64(3)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary',
    '{replica}'
 )
 ORDER BY (lesson_id, course_id);
 -- Создаем Materialized View
 DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary
 ON CLUSTER clickhouse_cluster;
 CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary
 ON CLUSTER clickhouse_cluster
 TO itikhomirov.lesson_popularity_summary
 AS
 SELECT
    flv.lesson_id AS lesson_id,
    l.title AS lesson_title,
    flv.course_id AS course_id,
    c.title AS course_title,
    count() AS total_views,
    uniqExact(flv.user_id) AS unique_users,
    min(flv.viewed_at) AS first_view,
    max(flv.viewed_at) AS last_view
 FROM itikhomirov.core_fact_lesson_views AS flv
 JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
 JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
 GROUP BY
    flv.lesson_id,
    l.title,
    flv.course_id,
    c.title;
 -- Вставляем данные в таблицу lesson_popularity_summary
 INSERT INTO itikhomirov.lesson_popularity_summary
 SELECT
    flv.lesson_id,
    l.title AS lesson_title,
    flv.course_id,
    c.title AS course_title,
    count() AS total_views,
    uniqExact(flv.user_id) AS unique_users,
    min(flv.viewed_at) AS first_view,
    max(flv.viewed_at) AS last_view
 FROM itikhomirov.core_fact_lesson_views AS flv
 JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
 JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
 GROUP BY
    flv.lesson_id,
    l.title,
    flv.course_id,
    c.title;
 -- Задание 3: Создай витрину inactive_users_summary
 -- Создаем пустую таблицу inactive_users_summary(структуру витрины)
 DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
 ON CLUSTER clickhouse_cluster;
 DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
 ON CLUSTER clickhouse_cluster;
 CREATE TABLE itikhomirov.inactive_users_summary
 ON CLUSTER clickhouse_cluster
 (
    user_id UInt32,
    name String,
    email String,
    age UInt8,
    registered_courses_count UInt32,
    registration_date DateTime64(3)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary',
    '{replica}'
 )
 ORDER BY (user_id);
 -- Создаем Materialized View
 DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary
 ON CLUSTER clickhouse_cluster;
 CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary
 ON CLUSTER clickhouse_cluster
 TO itikhomirov.inactive_users_summary
 AS
 WITH user_courses AS (
    SELECT
        e.user_id,
        countDistinct(e.course_id) AS registered_courses_count
    FROM itikhomirov.core_enrollments AS e
    GROUP BY
        e.user_id
 )
 SELECT
    u.user_id                                AS user_id,
    u.name                                   AS name,
    u.email                                  AS email,
    u.age                                    AS age,
    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
    u.registration_date                      AS registration_date
 FROM itikhomirov.core_user AS u
 LEFT JOIN user_courses AS uc
    ON u.user_id = uc.user_id
 WHERE u.user_id NOT IN (
    SELECT DISTINCT user_id
    FROM itikhomirov.core_fact_lesson_views
 );
 INSERT INTO itikhomirov.inactive_users_summary
 WITH user_courses AS (
    SELECT
        e.user_id,
        countDistinct(e.course_id) AS registered_courses_count
    FROM itikhomirov.core_enrollments AS e
    GROUP BY
        e.user_id
 )
 -- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров
 -- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views), 
 -- и к ним подмешивается уже готовый registered_courses_count.
 SELECT
    u.user_id AS user_id,
    u.name AS name,
    u.email AS email,
    u.age AS age,
    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
    u.registration_date AS registration_date
 FROM itikhomirov.core_user AS u
 LEFT JOIN user_courses AS uc
    ON u.user_id = uc.user_id
 WHERE u.user_id NOT IN (
    SELECT DISTINCT user_id
    FROM itikhomirov.core_fact_lesson_views
 )
 ORDER BY u.user_id;
 -- Задание 4: Создай витрину course_completion_rate
 CREATE TABLE itikhomirov.course_completion_rate
 ON CLUSTER clickhouse_cluster
 (
    user_id          UInt32,
    user_name        String,
    course_id        UInt32,
    course_title     String,
    lessons_in_course UInt32,
    lessons_viewed    UInt32,
    completion_rate   Float32
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/itikhomirov/course_completion_rate',
    '{replica}'
 )
 ORDER BY (user_id, course_id);
 -- Создаем Materialized View
 CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate
 ON CLUSTER clickhouse_cluster
 TO itikhomirov.course_completion_rate
 AS
 WITH
 lessons_per_course AS (
    SELECT
        course_id,
        countDistinct(lesson_id) AS lessons_in_course
    FROM itikhomirov.core_lesson
    GROUP BY course_id
 ),
 views_per_user_course AS (
    SELECT
        user_id,
        course_id,
        countDistinct(lesson_id) AS lessons_viewed
    FROM itikhomirov.core_fact_lesson_views
    GROUP BY user_id, course_id
 )
 SELECT
    u.user_id AS user_id,
    u.name AS user_name,
    c.course_id AS course_id,
    c.title AS course_title,
    lpc.lessons_in_course AS lessons_in_course,
    coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
    ROUND(
        COALESCE(
            NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
            / NULLIF(lpc.lessons_in_course * 1.0, 0),
            0
        ),
        2
    ) AS completion_rate
 FROM itikhomirov.core_user AS u
 JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
 JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
 JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
 LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
 ORDER BY
    u.user_id,
    c.course_id;
 -- Вставляем данные в таблицу pet.course_completion_rate
 INSERT INTO itikhomirov.course_completion_rate
 -- Создаем две временные таблицы:
 -- 1) Сколько уроков в каждом курсе
 WITH
 lessons_per_course AS (
    SELECT
        course_id,
        countDistinct(lesson_id) AS lessons_in_course
    FROM itikhomirov.core_lesson
    GROUP BY course_id
 ),
 -- 2) Сколько уроков каждого курса посмотрел пользователь
 views_per_user_course AS (
    SELECT
        user_id,
        course_id,
        countDistinct(lesson_id) AS lessons_viewed
    FROM itikhomirov.core_fact_lesson_views
    GROUP BY user_id, course_id
 )
 -- Основной запрос
 SELECT
    u.user_id AS user_id, 
    u.name AS user_name,
    c.course_id AS course_id,
    c.title AS course_title,
    lpc.lessons_in_course AS lessons_in_course,
    coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
    ROUND(
        COALESCE(
            NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
            / NULLIF(lpc.lessons_in_course * 1.0, 0),
            0
        ),
        2
    ) AS completion_rate
 FROM itikhomirov.core_user AS u
 JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
 JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
 JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
 LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
 ORDER BY
    u.user_id,
    c.course_id;
 -- Этап 3. Анализ и запросы
 -- Задание 5: Напиши SQL-запросы к витринам
 -- Уроки с хотя бы одним просмотром
 SELECT
    lesson_title,
    course_title,
    total_views
 FROM itikhomirov.lesson_popularity_summary
 WHERE total_views > 0
 ORDER BY lesson_title ASC;
 -- Топ‑3 урока внутри одного курса
 SELECT
    lesson_title,
    course_title,
    total_views
 FROM itikhomirov.lesson_popularity_summary
 WHERE course_title = 'SQL для начинающих'
 ORDER BY total_views DESC
 LIMIT 3;
 -- Пользователи, которые завершили курсы на 100%
 SELECT
    user_id,
    user_name,
    course_title,
    completion_rate
 FROM itikhomirov.course_completion_rate
 WHERE completion_rate = 1
 ORDER BY user_id;
 -- Этап 4. Тестирование и валидация
 -- Задание 6: Сравни данные с Greenplum**
 -- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
 -- 2. Сравни с ClickHouse-результатами:
 --    - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
 --    - Проверяй `NULL`, `0`, округления
 -- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
 SELECT
    sum(total_views)                 AS sum_total_views,
    avg(total_views)                 AS avg_total_views,
    min(total_views)                 AS min_total_views,
    max(total_views)                 AS max_total_views
 FROM itikhomirov.lesson_popularity_summary
 WHERE lesson_id IN (1, 3, 5);
 SELECT
    sum(total_views)                 AS sum_total_views,
    avg(total_views)                 AS avg_total_views,
    min(total_views)                 AS min_total_views,
    max(total_views)                 AS max_total_views
 FROM itikhomirov.lesson_popularity_summary
 WHERE lesson_id = 999;
 -- Отчёт в .doc файле