пет-проект полностью

2026-05-11 04:21:02 +04:00
parent c654840ffd
commit d280456d09
4 changed files with 377 additions and 0 deletions
@@ -0,0 +1,90 @@
 DROP TABLE IF EXISTS dsolovianov.core_fact_lesson_views ON CLUSTER clickhouse_cluster SYNC;
 DROP TABLE IF EXISTS dsolovianov.core_dim_user ON CLUSTER clickhouse_cluster SYNC;
 DROP TABLE IF EXISTS dsolovianov.core_dim_course ON CLUSTER clickhouse_cluster SYNC;
 DROP TABLE IF EXISTS dsolovianov.core_dim_lesson ON CLUSTER clickhouse_cluster SYNC;
 DROP TABLE IF EXISTS dsolovianov.core_dim_date ON CLUSTER clickhouse_cluster SYNC;
 DROP TABLE IF EXISTS dsolovianov.core_enrollments ON CLUSTER clickhouse_cluster SYNC;
 CREATE TABLE dsolovianov.core_dim_user
 ON CLUSTER clickhouse_cluster
 (
    user_id Int32,
    name String,
    age Int32,
    email String,
    registration_date Date
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/core_dim_user',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE TABLE dsolovianov.core_dim_course
 ON CLUSTER clickhouse_cluster
 (
    course_id Int32,
    title String,
    category String
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/core_dim_course',
    '{replica}'
 )
 ORDER BY course_id;
 CREATE TABLE dsolovianov.core_dim_lesson
 ON CLUSTER clickhouse_cluster
 (
    lesson_id Int32,
    title String,
    duration_min Int32,
    course_id Int32
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/core_dim_lesson',
    '{replica}'
 )
 ORDER BY lesson_id;
 CREATE TABLE dsolovianov.core_dim_date
 ON CLUSTER clickhouse_cluster
 (
    date Date,
    year Int32,
    month Int32,
    day Int32,
    weekday String
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/core_dim_date',
    '{replica}'
 )
 ORDER BY date;
 CREATE TABLE dsolovianov.core_enrollments
 ON CLUSTER clickhouse_cluster
 (
    user_id Int32,
    course_id Int32,
    enrolled_at DateTime64(3)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/core_enrollments',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE TABLE dsolovianov.core_fact_lesson_views
 ON CLUSTER clickhouse_cluster
 (
 	user_id Int32, 
    lesson_id Int32,
    course_id Int32,
    viewed_at DateTime64(3)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/core_fact_lesson_views',
    '{replica}'
 )
 ORDER BY user_id;
@@ -0,0 +1,186 @@
 -- Этап 2. Создание витринных таблиц --
 --  Витрина lesson_popularity_summary
 DROP TABLE IF EXISTS dsolovianov.lesson_popularity_summary ON CLUSTER clickhouse_cluster SYNC;
 CREATE TABLE dsolovianov.lesson_popularity_summary ON CLUSTER clickhouse_cluster (
    lesson_id UInt32,
    lesson_title String,
    course_id UInt32,
    course_title String,
    total_views UInt64,
    unique_users UInt64,
    first_view DateTime,
    last_view DateTime
 ) ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/lesson_popularity_summary',
    '{replica}'
 )
 ORDER BY (course_id, lesson_id);
 CREATE MATERIALIZED VIEW dsolovianov.lesson_popularity_summary_mv ON CLUSTER clickhouse_cluster
 TO dsolovianov.lesson_popularity_summary
 AS
 SELECT 
    l.lesson_id AS lesson_id,
    l.title AS lesson_title,
    l.course_id AS course_id,
    c.title AS course_title,
    COUNT(v.viewed_at) AS total_views,
    COUNT(DISTINCT v.user_id) AS unique_users,
    MIN(v.viewed_at) AS first_view,
    MAX(v.viewed_at) AS last_view
 FROM dsolovianov.core_fact_lesson_views v
 LEFT JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
 LEFT JOIN dsolovianov.core_dim_course c ON l.course_id = c.course_id
 GROUP BY l.lesson_id, l.title, l.course_id, c.title;
 INSERT INTO dsolovianov.lesson_popularity_summary
 SELECT 
    l.lesson_id AS lesson_id,
    l.title AS lesson_title,
    l.course_id AS course_id,
    c.title AS course_title,
    COUNT(v.viewed_at) AS total_views,
    COUNT(DISTINCT v.user_id) AS unique_users,
    MIN(v.viewed_at) AS first_view,
    MAX(v.viewed_at) AS last_view
 FROM dsolovianov.core_dim_lesson l
 JOIN dsolovianov.core_dim_course c ON l.course_id = c.course_id
 LEFT JOIN dsolovianov.core_fact_lesson_views v ON l.lesson_id = v.lesson_id
 GROUP BY l.lesson_id, l.title, l.course_id, c.title;
 --  Витрину inactive_users_summary
 DROP TABLE IF EXISTS dsolovianov.inactive_users_summary ON CLUSTER clickhouse_cluster SYNC;
 CREATE TABLE dsolovianov.inactive_users_summary ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    name String,
    email String,
    age UInt8,
    registered_courses_count UInt32,
    registration_date Date
 ) ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/inactive_users_summary',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE MATERIALIZED VIEW dsolovianov.inactive_users_summary_mv ON CLUSTER clickhouse_cluster
 TO dsolovianov.inactive_users_summary
 AS
 SELECT 
    u.user_id AS user_id,
    u.name AS name,
    u.email AS email,
    u.age AS age,
    COUNT(DISTINCT e.course_id) AS registered_courses_count,
    u.registration_date AS registration_date
 FROM dsolovianov.core_dim_user u
 LEFT JOIN dsolovianov.core_enrollments e ON u.user_id = e.user_id
 WHERE u.user_id NOT IN (SELECT DISTINCT user_id FROM dsolovianov.core_fact_lesson_views)
 GROUP BY u.user_id, u.name, u.email, u.age, u.registration_date;
 INSERT INTO dsolovianov.inactive_users_summary
 SELECT 
    u.user_id AS user_id,
    u.name AS name,
    u.email AS email,
    u.age AS age,
    COUNT(DISTINCT e.course_id) AS registered_courses_count,
    u.registration_date AS registration_date
 FROM dsolovianov.core_dim_user u
 LEFT JOIN dsolovianov.core_enrollments e ON u.user_id = e.user_id
 WHERE u.user_id NOT IN (SELECT DISTINCT user_id FROM dsolovianov.core_fact_lesson_views)
 GROUP BY u.user_id, u.name, u.email, u.age, u.registration_date;
 -- Витрина course_completion_rate
 DROP TABLE IF EXISTS dsolovianov.course_completion_rate ON CLUSTER clickhouse_cluster SYNC;
 CREATE TABLE dsolovianov.course_completion_rate ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    user_name String,
    course_id UInt32,
    course_title String,
    lessons_in_course UInt32,
    lessons_viewed UInt32,
    completion_rate Float32
 ) ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/dsolovianov/course_completion_rate',
    '{replica}'
 )
 ORDER BY (user_id, course_id);
 DROP TABLE IF EXISTS dsolovianov.course_completion_rate_mv ON CLUSTER clickhouse_cluster SYNC;
 CREATE MATERIALIZED VIEW dsolovianov.course_completion_rate_mv ON CLUSTER clickhouse_cluster
 TO dsolovianov.course_completion_rate
 AS
 SELECT 
    e.user_id AS user_id,
    u.name AS user_name,
    e.course_id AS course_id,
    c.title AS course_title,
    (
        SELECT COUNT(*)
        FROM dsolovianov.core_dim_lesson l
        WHERE l.course_id = e.course_id
    ) AS lessons_in_course,
    (
        SELECT COUNT(DISTINCT v.lesson_id)
        FROM dsolovianov.core_fact_lesson_views v
        JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
        WHERE v.user_id = e.user_id AND l.course_id = e.course_id
    ) AS lessons_viewed,
    (
        SELECT COUNT(DISTINCT v.lesson_id)
        FROM dsolovianov.core_fact_lesson_views v
        JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
        WHERE v.user_id = e.user_id AND l.course_id = e.course_id
    ) / (
        SELECT COUNT(*)
        FROM dsolovianov.core_dim_lesson l
        WHERE l.course_id = e.course_id
    ) AS completion_rate
 FROM dsolovianov.core_enrollments e
 JOIN dsolovianov.core_dim_user u ON e.user_id = u.user_id
 JOIN dsolovianov.core_dim_course c ON e.course_id = c.course_id
 GROUP BY e.user_id, u.name, e.course_id, c.title;
 INSERT INTO dsolovianov.course_completion_rate
 SELECT 
    e.user_id AS user_id,
    u.name AS user_name,
    e.course_id AS course_id,
    c.title AS course_title,
    (
        SELECT COUNT(*)
        FROM dsolovianov.core_dim_lesson l
        WHERE l.course_id = e.course_id
    ) AS lessons_in_course,
    COALESCE(
        (
            SELECT COUNT(DISTINCT v.lesson_id)
            FROM dsolovianov.core_fact_lesson_views v
            JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
            WHERE v.user_id = e.user_id AND l.course_id = e.course_id
        ), 0
    ) AS lessons_viewed,
    COALESCE(
        (
            SELECT COUNT(DISTINCT v.lesson_id)
            FROM dsolovianov.core_fact_lesson_views v
            JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
            WHERE v.user_id = e.user_id AND l.course_id = e.course_id
        ), 0
    ) / (
        SELECT COUNT(*)
        FROM dsolovianov.core_dim_lesson l
        WHERE l.course_id = e.course_id
    ) AS completion_rate
 FROM dsolovianov.core_enrollments e
 JOIN dsolovianov.core_dim_user u ON e.user_id = u.user_id
 JOIN dsolovianov.core_dim_course c ON e.course_id = c.course_id
 GROUP BY e.user_id, u.name, e.course_id, c.title;
@@ -0,0 +1,88 @@
 -- Этап 3. Анализ и запросы - Напиши SQL-запросы к витринам -- Напиши свои вариации для практики--
 -- Топ-5 самых просматриваемых уроков
 SELECT 
    lesson_title,
    course_title,
    total_views
 FROM lesson_popularity_summary
 ORDER BY total_views DESC
 LIMIT 5;
 -- Неактивные пользователи, записавшиеся на курсы
 SELECT 
    name,
    email,
    registered_courses_count
 FROM inactive_users_summary
 WHERE registered_courses_count > 0
 ORDER BY registration_date DESC;
 -- Курсы с самым высоким процентом завершения
 SELECT 
    course_title,
    AVG(completion_rate) AS avg_completion
 FROM course_completion_rate
 GROUP BY course_title
 ORDER BY avg_completion DESC
 LIMIT 5;
 -- Мои вариации --
 -- Среднее количество учеников на урок по каждому курсу
 SELECT 
    course_title,
    ROUND(AVG(unique_users), 2) AS avg_users_lesson,
    SUM(total_views) AS total_course_views
 FROM dsolovianov.lesson_popularity_summary
 GROUP BY course_title
 ORDER BY avg_users_lesson DESC;
 -- Возрастное распределение неактивных пользователей
 SELECT 
    CASE 
        WHEN age < 25 THEN 'до 25'
        WHEN age BETWEEN 25 AND 30 THEN '25-30'
        WHEN age BETWEEN 31 AND 35 THEN '31-35'
        ELSE '35+'
    END AS age_group,
    COUNT(*) AS inactive_users_count,
    AVG(registered_courses_count) AS avg_courses_enrolled
 FROM dsolovianov.inactive_users_summary
 GROUP BY age_group
 ORDER BY age_group;
 -- Пользователи, которые начали, но не закончили курс (completion_rate между 0 и 1)
 SELECT 
    user_name,
    course_title,
    lessons_viewed,
    lessons_in_course,
    ROUND(completion_rate * 100, 0) AS no_completion,
    (lessons_in_course - lessons_viewed) AS remaining_lessons
 FROM dsolovianov.course_completion_rate
 WHERE completion_rate BETWEEN 0.01 AND 0.99
 ORDER BY completion_rate DESC, user_name;
 -- Прогресс каждого пользователя по курсам (с указанием статуса)
 SELECT 
    user_name,
    course_title,
    lessons_viewed,
    lessons_in_course,
    ROUND(completion_rate * 100, 0) AS progress_percent,
    CASE 
        WHEN completion_rate = 1.0 THEN 'Пройден'
        WHEN completion_rate = 0 THEN 'Не начат'
        WHEN completion_rate >= 0.5 THEN 'В процессе (>50%)'
        ELSE 'В процессе (<50%)'
    END AS status
 FROM dsolovianov.course_completion_rate
 ORDER BY user_name, completion_rate DESC;
@@ -0,0 +1,13 @@
 -- Этап 4. Тестирование и валидация --
 /*
 ### **Задание 6: Сравни данные с Greenplum**
 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
 2. Сравни с ClickHouse-результатами:
    - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
    - Проверяй `NULL`, `0`, округления
 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
 */
 -- Ничего не понятно по этому этапу. У меня в GP нет таблиц "_summary_source"