пет-проект полностью

dsolovianov/clickhouse/Pet-Project/Этап 1. Загрузка тестовых данных.sql

@@ -0,0 +1,90 @@
+DROP TABLE IF EXISTS dsolovianov.core_fact_lesson_views ON CLUSTER clickhouse_cluster SYNC;
+DROP TABLE IF EXISTS dsolovianov.core_dim_user ON CLUSTER clickhouse_cluster SYNC;
+DROP TABLE IF EXISTS dsolovianov.core_dim_course ON CLUSTER clickhouse_cluster SYNC;
+DROP TABLE IF EXISTS dsolovianov.core_dim_lesson ON CLUSTER clickhouse_cluster SYNC;
+DROP TABLE IF EXISTS dsolovianov.core_dim_date ON CLUSTER clickhouse_cluster SYNC;
+DROP TABLE IF EXISTS dsolovianov.core_enrollments ON CLUSTER clickhouse_cluster SYNC;
+
+CREATE TABLE dsolovianov.core_dim_user
+ON CLUSTER clickhouse_cluster
+(
+    user_id Int32,
+    name String,
+    age Int32,
+    email String,
+    registration_date Date
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/core_dim_user',
+    '{replica}'
+)
+ORDER BY user_id;
+
+CREATE TABLE dsolovianov.core_dim_course
+ON CLUSTER clickhouse_cluster
+(
+    course_id Int32,
+    title String,
+    category String
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/core_dim_course',
+    '{replica}'
+)
+ORDER BY course_id;
+
+CREATE TABLE dsolovianov.core_dim_lesson
+ON CLUSTER clickhouse_cluster
+(
+    lesson_id Int32,
+    title String,
+    duration_min Int32,
+    course_id Int32
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/core_dim_lesson',
+    '{replica}'
+)
+ORDER BY lesson_id;
+
+CREATE TABLE dsolovianov.core_dim_date
+ON CLUSTER clickhouse_cluster
+(
+    date Date,
+    year Int32,
+    month Int32,
+    day Int32,
+    weekday String
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/core_dim_date',
+    '{replica}'
+)
+ORDER BY date;
+
+CREATE TABLE dsolovianov.core_enrollments
+ON CLUSTER clickhouse_cluster
+(
+    user_id Int32,
+    course_id Int32,
+    enrolled_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/core_enrollments',
+    '{replica}'
+)
+ORDER BY user_id;
+
+CREATE TABLE dsolovianov.core_fact_lesson_views
+ON CLUSTER clickhouse_cluster
+(
+ 	user_id Int32, 
+    lesson_id Int32,
+    course_id Int32,
+    viewed_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/core_fact_lesson_views',
+    '{replica}'
+)
+ORDER BY user_id;

dsolovianov/clickhouse/Pet-Project/Этап 2. Создание витринных таблиц.sql

@@ -0,0 +1,186 @@
+-- Этап 2. Создание витринных таблиц --
+
+--  Витрина lesson_popularity_summary
+DROP TABLE IF EXISTS dsolovianov.lesson_popularity_summary ON CLUSTER clickhouse_cluster SYNC;
+
+CREATE TABLE dsolovianov.lesson_popularity_summary ON CLUSTER clickhouse_cluster (
+    lesson_id UInt32,
+    lesson_title String,
+    course_id UInt32,
+    course_title String,
+    total_views UInt64,
+    unique_users UInt64,
+    first_view DateTime,
+    last_view DateTime
+) ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/lesson_popularity_summary',
+    '{replica}'
+)
+ORDER BY (course_id, lesson_id);
+
+CREATE MATERIALIZED VIEW dsolovianov.lesson_popularity_summary_mv ON CLUSTER clickhouse_cluster
+TO dsolovianov.lesson_popularity_summary
+AS
+SELECT 
+    l.lesson_id AS lesson_id,
+    l.title AS lesson_title,
+    l.course_id AS course_id,
+    c.title AS course_title,
+    COUNT(v.viewed_at) AS total_views,
+    COUNT(DISTINCT v.user_id) AS unique_users,
+    MIN(v.viewed_at) AS first_view,
+    MAX(v.viewed_at) AS last_view
+FROM dsolovianov.core_fact_lesson_views v
+LEFT JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
+LEFT JOIN dsolovianov.core_dim_course c ON l.course_id = c.course_id
+GROUP BY l.lesson_id, l.title, l.course_id, c.title;
+
+INSERT INTO dsolovianov.lesson_popularity_summary
+SELECT 
+    l.lesson_id AS lesson_id,
+    l.title AS lesson_title,
+    l.course_id AS course_id,
+    c.title AS course_title,
+    COUNT(v.viewed_at) AS total_views,
+    COUNT(DISTINCT v.user_id) AS unique_users,
+    MIN(v.viewed_at) AS first_view,
+    MAX(v.viewed_at) AS last_view
+FROM dsolovianov.core_dim_lesson l
+JOIN dsolovianov.core_dim_course c ON l.course_id = c.course_id
+LEFT JOIN dsolovianov.core_fact_lesson_views v ON l.lesson_id = v.lesson_id
+GROUP BY l.lesson_id, l.title, l.course_id, c.title;
+
+--  Витрину inactive_users_summary
+
+DROP TABLE IF EXISTS dsolovianov.inactive_users_summary ON CLUSTER clickhouse_cluster SYNC;
+
+CREATE TABLE dsolovianov.inactive_users_summary ON CLUSTER clickhouse_cluster (
+    user_id UInt32,
+    name String,
+    email String,
+    age UInt8,
+    registered_courses_count UInt32,
+    registration_date Date
+) ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/inactive_users_summary',
+    '{replica}'
+)
+ORDER BY user_id;
+
+CREATE MATERIALIZED VIEW dsolovianov.inactive_users_summary_mv ON CLUSTER clickhouse_cluster
+TO dsolovianov.inactive_users_summary
+AS
+SELECT 
+    u.user_id AS user_id,
+    u.name AS name,
+    u.email AS email,
+    u.age AS age,
+    COUNT(DISTINCT e.course_id) AS registered_courses_count,
+    u.registration_date AS registration_date
+FROM dsolovianov.core_dim_user u
+LEFT JOIN dsolovianov.core_enrollments e ON u.user_id = e.user_id
+WHERE u.user_id NOT IN (SELECT DISTINCT user_id FROM dsolovianov.core_fact_lesson_views)
+GROUP BY u.user_id, u.name, u.email, u.age, u.registration_date;
+
+INSERT INTO dsolovianov.inactive_users_summary
+SELECT 
+    u.user_id AS user_id,
+    u.name AS name,
+    u.email AS email,
+    u.age AS age,
+    COUNT(DISTINCT e.course_id) AS registered_courses_count,
+    u.registration_date AS registration_date
+FROM dsolovianov.core_dim_user u
+LEFT JOIN dsolovianov.core_enrollments e ON u.user_id = e.user_id
+WHERE u.user_id NOT IN (SELECT DISTINCT user_id FROM dsolovianov.core_fact_lesson_views)
+GROUP BY u.user_id, u.name, u.email, u.age, u.registration_date;
+
+-- Витрина course_completion_rate
+
+DROP TABLE IF EXISTS dsolovianov.course_completion_rate ON CLUSTER clickhouse_cluster SYNC;
+
+CREATE TABLE dsolovianov.course_completion_rate ON CLUSTER clickhouse_cluster (
+    user_id UInt32,
+    user_name String,
+    course_id UInt32,
+    course_title String,
+    lessons_in_course UInt32,
+    lessons_viewed UInt32,
+    completion_rate Float32
+) ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/dsolovianov/course_completion_rate',
+    '{replica}'
+)
+ORDER BY (user_id, course_id);
+
+DROP TABLE IF EXISTS dsolovianov.course_completion_rate_mv ON CLUSTER clickhouse_cluster SYNC;
+
+CREATE MATERIALIZED VIEW dsolovianov.course_completion_rate_mv ON CLUSTER clickhouse_cluster
+TO dsolovianov.course_completion_rate
+AS
+SELECT 
+    e.user_id AS user_id,
+    u.name AS user_name,
+    e.course_id AS course_id,
+    c.title AS course_title,
+    (
+        SELECT COUNT(*)
+        FROM dsolovianov.core_dim_lesson l
+        WHERE l.course_id = e.course_id
+    ) AS lessons_in_course,
+    (
+        SELECT COUNT(DISTINCT v.lesson_id)
+        FROM dsolovianov.core_fact_lesson_views v
+        JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
+        WHERE v.user_id = e.user_id AND l.course_id = e.course_id
+    ) AS lessons_viewed,
+    (
+        SELECT COUNT(DISTINCT v.lesson_id)
+        FROM dsolovianov.core_fact_lesson_views v
+        JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
+        WHERE v.user_id = e.user_id AND l.course_id = e.course_id
+    ) / (
+        SELECT COUNT(*)
+        FROM dsolovianov.core_dim_lesson l
+        WHERE l.course_id = e.course_id
+    ) AS completion_rate
+FROM dsolovianov.core_enrollments e
+JOIN dsolovianov.core_dim_user u ON e.user_id = u.user_id
+JOIN dsolovianov.core_dim_course c ON e.course_id = c.course_id
+GROUP BY e.user_id, u.name, e.course_id, c.title;
+
+INSERT INTO dsolovianov.course_completion_rate
+SELECT 
+    e.user_id AS user_id,
+    u.name AS user_name,
+    e.course_id AS course_id,
+    c.title AS course_title,
+    (
+        SELECT COUNT(*)
+        FROM dsolovianov.core_dim_lesson l
+        WHERE l.course_id = e.course_id
+    ) AS lessons_in_course,
+    COALESCE(
+        (
+            SELECT COUNT(DISTINCT v.lesson_id)
+            FROM dsolovianov.core_fact_lesson_views v
+            JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
+            WHERE v.user_id = e.user_id AND l.course_id = e.course_id
+        ), 0
+    ) AS lessons_viewed,
+    COALESCE(
+        (
+            SELECT COUNT(DISTINCT v.lesson_id)
+            FROM dsolovianov.core_fact_lesson_views v
+            JOIN dsolovianov.core_dim_lesson l ON v.lesson_id = l.lesson_id
+            WHERE v.user_id = e.user_id AND l.course_id = e.course_id
+        ), 0
+    ) / (
+        SELECT COUNT(*)
+        FROM dsolovianov.core_dim_lesson l
+        WHERE l.course_id = e.course_id
+    ) AS completion_rate
+FROM dsolovianov.core_enrollments e
+JOIN dsolovianov.core_dim_user u ON e.user_id = u.user_id
+JOIN dsolovianov.core_dim_course c ON e.course_id = c.course_id
+GROUP BY e.user_id, u.name, e.course_id, c.title;

dsolovianov/clickhouse/Pet-Project/Этап 3. Анализ и запросы.sql

@@ -0,0 +1,88 @@
+-- Этап 3. Анализ и запросы - Напиши SQL-запросы к витринам -- Напиши свои вариации для практики--
+-- Топ-5 самых просматриваемых уроков
+
+SELECT 
+    lesson_title,
+    course_title,
+    total_views
+FROM lesson_popularity_summary
+ORDER BY total_views DESC
+LIMIT 5;
+
+-- Неактивные пользователи, записавшиеся на курсы
+
+SELECT 
+    name,
+    email,
+    registered_courses_count
+FROM inactive_users_summary
+WHERE registered_courses_count > 0
+ORDER BY registration_date DESC;
+
+-- Курсы с самым высоким процентом завершения
+
+SELECT 
+    course_title,
+    AVG(completion_rate) AS avg_completion
+FROM course_completion_rate
+GROUP BY course_title
+ORDER BY avg_completion DESC
+LIMIT 5;
+
+-- Мои вариации --
+
+-- Среднее количество учеников на урок по каждому курсу
+
+SELECT 
+    course_title,
+    ROUND(AVG(unique_users), 2) AS avg_users_lesson,
+    SUM(total_views) AS total_course_views
+FROM dsolovianov.lesson_popularity_summary
+GROUP BY course_title
+ORDER BY avg_users_lesson DESC;
+
+-- Возрастное распределение неактивных пользователей
+
+SELECT 
+    CASE 
+        WHEN age < 25 THEN 'до 25'
+        WHEN age BETWEEN 25 AND 30 THEN '25-30'
+        WHEN age BETWEEN 31 AND 35 THEN '31-35'
+        ELSE '35+'
+    END AS age_group,
+    COUNT(*) AS inactive_users_count,
+    AVG(registered_courses_count) AS avg_courses_enrolled
+FROM dsolovianov.inactive_users_summary
+GROUP BY age_group
+ORDER BY age_group;
+
+-- Пользователи, которые начали, но не закончили курс (completion_rate между 0 и 1)
+
+SELECT 
+    user_name,
+    course_title,
+    lessons_viewed,
+    lessons_in_course,
+    ROUND(completion_rate * 100, 0) AS no_completion,
+    (lessons_in_course - lessons_viewed) AS remaining_lessons
+FROM dsolovianov.course_completion_rate
+WHERE completion_rate BETWEEN 0.01 AND 0.99
+ORDER BY completion_rate DESC, user_name;
+
+-- Прогресс каждого пользователя по курсам (с указанием статуса)
+
+SELECT 
+    user_name,
+    course_title,
+    lessons_viewed,
+    lessons_in_course,
+    ROUND(completion_rate * 100, 0) AS progress_percent,
+    CASE 
+        WHEN completion_rate = 1.0 THEN 'Пройден'
+        WHEN completion_rate = 0 THEN 'Не начат'
+        WHEN completion_rate >= 0.5 THEN 'В процессе (>50%)'
+        ELSE 'В процессе (<50%)'
+    END AS status
+FROM dsolovianov.course_completion_rate
+ORDER BY user_name, completion_rate DESC;
+

dsolovianov/clickhouse/Pet-Project/Этап 4. Тестирование и валидация.sql

@@ -0,0 +1,13 @@
+-- Этап 4. Тестирование и валидация --
+
+/*
+### **Задание 6: Сравни данные с Greenplum**
+
+1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
+2. Сравни с ClickHouse-результатами:
+    - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
+    - Проверяй `NULL`, `0`, округления
+3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
+*/
+
+-- Ничего не понятно по этому этапу. У меня в GP нет таблиц "_summary_source"