Merge pull request 'migration to ckickhouse' (#26) from OS-220 into main

Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/26 Reviewed-by: aleksey bogatyrev <abogatyrev@example.com>
2026-01-07 21:13:50 +04:00
parent d514c04a88 df53d48e67
commit d271077b42
2 changed files with 279 additions and 1 deletions
@@ -0,0 +1,278 @@
 CREATE TABLE ybirk.core_user ON CLUSTER clickhouse_cluster
 (
    user_id Int32,
    name String,
    age Int32,
    email String,
    registration_date Date
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ybirk/core_user',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE TABLE ybirk.core_course ON CLUSTER clickhouse_cluster
 (
    course_id Int32,
    title String,
    category String
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ybirk/core_course',
    '{replica}'
 )
 ORDER BY course_id;
 CREATE TABLE ybirk.core_lesson ON CLUSTER clickhouse_cluster
 (
    lesson_id Int32,
    title String,
    duration_min Int32,
    course_id Int32
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ybirk/core_lesson',
    '{replica}'
 )
 ORDER BY lesson_id;
 CREATE TABLE ybirk.core_fact_lesson_views ON CLUSTER clickhouse_cluster
 (
    user_id Int32, 
    lesson_id Int32,
    course_id Int32,
    viewed_at DateTime64(3)
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ybirk/core_fact_lesson_views',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE TABLE ybirk.core_enrollments ON CLUSTER clickhouse_cluster
 (
    id Int32,
    user_id Int32,
    course_id Int32,
    enrolled_at Date
 )
 ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ybirk/core_enrollments',
    '{replica}'
 )
 ORDER BY user_id;
 --------------------------------
 CREATE TABLE ybirk.inactive_users_summary ON CLUSTER clickhouse_cluster
 (
    user_id UInt32,
    name String,
    email String,
    age UInt8,
    registered_courses_count UInt32,
    registration_date DateTime64(3, 'UTC')
 )
 ENGINE = ReplicatedAggregatingMergeTree(
    '/clickhouse/tables/{shard}/ybirk/inactive_users_summary',
    '{replica}'
 )
 ORDER BY user_id;
 CREATE TABLE ybirk.lesson_popularity_summary ON CLUSTER clickhouse_cluster
 (
    lesson_id UInt32,
    lesson_title String,
    course_id UInt32,
    course_title String,
    total_views UInt32,
    unique_users UInt32,
    first_view DateTime64(3, 'UTC'),
    last_view DateTime64(3, 'UTC')
 )
 ENGINE = ReplicatedAggregatingMergeTree(
    '/clickhouse/tables/{shard}/ybirk/lesson_popularity_summary',
    '{replica}'
 )
 ORDER BY (lesson_id, course_id);
 CREATE TABLE ybirk.course_completion_rate ON CLUSTER clickhouse_cluster
 (
    user_id UInt32,
    user_name String,
    course_id UInt32,
    course_title String,
    lessons_in_course UInt32,
    lessons_viewed UInt32,
    completion_rate Float32
 )
 ENGINE = ReplicatedAggregatingMergeTree(
    '/clickhouse/tables/{shard}/ybirk/course_completion_rate',
    '{replica}'
 )
 ORDER BY (user_id, course_id);
 --------------------------------------
 CREATE MATERIALIZED VIEW ybirk.mv_lesson_popularity_summary ON CLUSTER clickhouse_cluster
 TO ybirk.lesson_popularity_summary
 AS
 SELECT
    flv.lesson_id AS lesson_id,
    l.title AS lesson_title,
    flv.course_id AS course_id,
    c.title AS course_title,
    count() AS total_views,
    uniqExact(flv.user_id) AS unique_users,
    min(flv.viewed_at) AS first_view,
    max(flv.viewed_at) AS last_view
 FROM ybirk.core_fact_lesson_views AS flv
 JOIN ybirk.core_lesson AS l ON flv.lesson_id = l.lesson_id
 JOIN ybirk.core_course AS c ON flv.course_id = c.course_id
 GROUP BY
    flv.lesson_id,
    l.title,
    flv.course_id,
    c.title;
 DROP VIEW IF EXISTS ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster;
 CREATE MATERIALIZED VIEW ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster
 TO ybirk.inactive_users_summary
 AS
 SELECT
    u.user_id AS user_id,
    u.name AS name,
    u.email AS email,
    u.age AS age,
    countDistinct(e.course_id) AS registered_courses_count,
    u.registration_date AS registration_date
 FROM ybirk.core_user AS u
 LEFT JOIN ybirk.core_enrollments AS e ON u.user_id = e.user_id
 LEFT ANTI JOIN ybirk.core_fact_lesson_views AS flv ON u.user_id = flv.user_id
 GROUP BY
    u.user_id,
    u.name,
    u.email,
    u.age,
    u.registration_date;
 CREATE MATERIALIZED VIEW ybirk.mv_course_completion_rate ON CLUSTER clickhouse_cluster
 TO ybirk.course_completion_rate
 AS
 SELECT
    u.user_id AS user_id,
    u.name AS user_name,
    c.course_id AS course_id,
    c.title AS course_title,
    countDistinct(l.lesson_id) AS lessons_in_course,
    countDistinct(flv.lesson_id) AS lessons_viewed,
    if(countDistinct(l.lesson_id) = 0, 0,
       round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id), 2)
    ) AS completion_rate
 FROM ybirk.core_enrollments AS e
 JOIN ybirk.core_user AS u ON e.user_id = u.user_id
 JOIN ybirk.core_course AS c ON e.course_id = c.course_id
 JOIN ybirk.core_lesson AS l ON c.course_id = l.course_id
 LEFT JOIN ybirk.core_fact_lesson_views AS flv
    ON flv.user_id = e.user_id
    AND flv.course_id = e.course_id
 GROUP BY
    u.user_id,
    u.name,
    c.course_id,
    c.title;
 ----------------------------------------------
 INSERT INTO ybirk.core_user (user_id, name, age, email, registration_date) VALUES
 (1, 'Alice', 25, 'alice@mail.com', '2023-01-10'),
 (2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'),
 (3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'),
 (4, 'Diana', 28, 'diana@mail.com', '2023-03-01'),
 (5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10');
 INSERT INTO ybirk.core_course (course_id, title, category) VALUES
 (1, 'SQL для начинающих', 'data'),
 (2, 'Python для анализа данных', 'programming'),
 (3, 'BI с нуля', 'business');
 INSERT INTO ybirk.core_lesson (lesson_id, course_id, title, duration_min) VALUES
 (1, 1, 'SELECT и FROM', 10),
 (2, 1, 'JOIN', 15),
 (3, 2, 'Pandas', 20),
 (4, 3, 'Основы BI', 12),
 (5, 3, 'Метрики и дашборды', 18);
 INSERT INTO ybirk.core_fact_lesson_views (user_id, lesson_id, course_id, viewed_at) VALUES
 (1, 1, 1, '2023-01-16 10:00:00'),
 (1, 2, 1, '2023-01-16 10:15:00'),
 (2, 1, 1, '2023-01-21 09:00:00'),
 (3, 3, 2, '2023-03-06 12:00:00'),
 (4, 4, 3, '2023-04-02 10:00:00'),
 (4, 5, 3, '2023-04-03 10:00:00');
 INSERT INTO ybirk.core_enrollments (id, user_id, course_id, enrolled_at) VALUES
 (1, 1, 1, '2023-01-15'),
 (2, 1, 2, '2023-02-01'),
 (3, 2, 1, '2023-01-20'),
 (4, 3, 2, '2023-03-05'),
 (5, 4, 3, '2023-04-01');    
 --------------------------------------------------
 --Примеры
 SELECT 
    lesson_title,
    course_title,
    total_views
 FROM ybirk.mv_lesson_popularity_summary
 ORDER BY total_views DESC
 LIMIT 5;
 SELECT 
    name,
    email,
    registered_courses_count
 FROM ybirk.inactive_users_summary
 WHERE registered_courses_count > 0
 ORDER BY registration_date DESC;
 SELECT 
    course_title,
    AVG(completion_rate) AS avg_completion
 FROM ybirk.course_completion_rate
 GROUP BY course_title
 ORDER BY avg_completion DESC
 LIMIT 5;
 -- ЗАпросы
 -- самый популярный курс
 SELECT 
    course_title, 
    SUM(total_views) as sum_views
 FROM ybirk.mv_lesson_popularity_summary
 GROUP BY course_title
 ORDER BY sum_views DESC;
 -- Кто застрял
 SELECT 
    user_name, 
    course_title,
    completion_rate
 FROM ybirk.mv_course_completion_rate
 WHERE completion_rate > 0 AND completion_rate < 1.0;
 -- тест3
 SELECT 
    t1.user_name,
    t1.course_title,
    t2.email
 FROM ybirk.mv_course_completion_rate AS t1
 JOIN ybirk.mv_inactive_users_summary AS t2 
    ON t1.user_id = t2.user_id
 WHERE t1.completion_rate = 1.0;