dmitry_medvedev/homework
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

миграция таблиц на ReplicatedMergeTree с поддержкой кластера

Browse Source 
добавил примеры для проверки
This commit is contained in:
Birk
2026-01-03 16:24:51 +03:00
parent cc04025ade
commit 4f017d0efd
1 changed files with 138 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ybirk/clickhouse/task_220_&_221.sql
+138 -54
View File
@@ -1,94 +1,121 @@
CREATE TABLE ybirk.core_user ( CREATE TABLE ybirk.core_user ON CLUSTER clickhouse_cluster
(
user_id Int32, user_id Int32,
name String, name String,
age Int32, age Int32,
email String, email String,
registration_date Date registration_date Date
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_user',
'{replica}'
)
ORDER BY user_id; ORDER BY user_id;
CREATE TABLE ybirk.core_course ( CREATE TABLE ybirk.core_course ON CLUSTER clickhouse_cluster
(
course_id Int32, course_id Int32,
title String, title String,
category String category String
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_course',
'{replica}'
)
ORDER BY course_id; ORDER BY course_id;
CREATE TABLE ybirk.core_lesson ( CREATE TABLE ybirk.core_lesson ON CLUSTER clickhouse_cluster
(
lesson_id Int32, lesson_id Int32,
title String, title String,
duration_min Int32, duration_min Int32,
course_id Int32 course_id Int32
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_lesson',
'{replica}'
)
ORDER BY lesson_id; ORDER BY lesson_id;
CREATE TABLE ybirk.core_fact_lesson_views ( CREATE TABLE ybirk.core_fact_lesson_views ON CLUSTER clickhouse_cluster
(
user_id Int32, user_id Int32,
lesson_id Int32, lesson_id Int32,
course_id Int32, course_id Int32,
viewed_at DateTime64(3) viewed_at DateTime64(3)
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_fact_lesson_views',
'{replica}'
)
ORDER BY user_id; ORDER BY user_id;
CREATE TABLE ybirk.core_enrollments ( CREATE TABLE ybirk.core_enrollments ON CLUSTER clickhouse_cluster
(
id Int32, id Int32,
user_id Int32, user_id Int32,
course_id Int32, course_id Int32,
enrolled_at Date enrolled_at Date
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_enrollments',
'{replica}'
)
ORDER BY user_id; ORDER BY user_id;
---------------------------------- --------------------------------
CREATE TABLE ybirk.inactive_users_summary CREATE TABLE ybirk.inactive_users_summary ON CLUSTER clickhouse_cluster
( (
user_id UInt32, user_id UInt32,
name String, name String,
email String, email String,
age UInt8, age UInt8,
registered_courses_count UInt32, registered_courses_count UInt32,
registration_date DateTime64(3, 'UTC') registration_date DateTime64(3, 'UTC')
) )
ENGINE = AggregatingMergeTree ENGINE = ReplicatedAggregatingMergeTree(
ORDER BY (user_id); '/clickhouse/tables/{shard}/ybirk/inactive_users_summary',
'{replica}'
)
ORDER BY user_id;
CREATE TABLE ybirk.lesson_popularity_summary ON CLUSTER clickhouse_cluster
CREATE TABLE ybirk.lesson_popularity_summary
( (
lesson_id UInt32, lesson_id UInt32,
lesson_title String, lesson_title String,
course_id UInt32, course_id UInt32,
course_title String, course_title String,
total_views UInt32, total_views UInt32,
unique_users UInt32, unique_users UInt32,
first_view DateTime64(3, 'UTC'), first_view DateTime64(3, 'UTC'),
last_view DateTime64(3, 'UTC') last_view DateTime64(3, 'UTC')
)
ENGINE = ReplicatedAggregatingMergeTree(
'/clickhouse/tables/{shard}/ybirk/lesson_popularity_summary',
'{replica}'
) )
ENGINE = AggregatingMergeTree
ORDER BY (lesson_id, course_id); ORDER BY (lesson_id, course_id);
CREATE TABLE ybirk.course_completion_rate ON CLUSTER clickhouse_cluster
CREATE TABLE ybirk.course_completion_rate
( (
user_id UInt32, user_id UInt32,
user_name String, user_name String,
course_id UInt32, course_id UInt32,
course_title String, course_title String,
lessons_in_course UInt32, lessons_in_course UInt32,
lessons_viewed UInt32, lessons_viewed UInt32,
completion_rate Float32 completion_rate Float32
)
ENGINE = ReplicatedAggregatingMergeTree(
'/clickhouse/tables/{shard}/ybirk/course_completion_rate',
'{replica}'
) )
ENGINE = AggregatingMergeTree
ORDER BY (user_id, course_id); ORDER BY (user_id, course_id);
------------------------------------------- --------------------------------------
CREATE MATERIALIZED VIEW ybirk.mv_lesson_popularity_summary CREATE MATERIALIZED VIEW ybirk.mv_lesson_popularity_summary ON CLUSTER clickhouse_cluster
TO ybirk.lesson_popularity_summary TO ybirk.lesson_popularity_summary
AS AS
SELECT SELECT
@@ -101,15 +128,15 @@ SELECT
min(flv.viewed_at) AS first_view, min(flv.viewed_at) AS first_view,
max(flv.viewed_at) AS last_view max(flv.viewed_at) AS last_view
FROM ybirk.core_fact_lesson_views AS flv FROM ybirk.core_fact_lesson_views AS flv
JOIN ybirk.core_lesson AS l ON flv.lesson_id = l.lesson_id JOIN ybirk.core_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN ybirk.core_course AS c ON flv.course_id = c.course_id JOIN ybirk.core_course AS c ON flv.course_id = c.course_id
GROUP BY GROUP BY
flv.lesson_id, flv.lesson_id,
l.title, l.title,
flv.course_id, flv.course_id,
c.title; c.title;
CREATE MATERIALIZED VIEW ybirk.mv_inactive_users_summary CREATE MATERIALIZED VIEW ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster
TO ybirk.inactive_users_summary TO ybirk.inactive_users_summary
AS AS
SELECT SELECT
@@ -132,7 +159,7 @@ GROUP BY
u.age, u.age,
u.registration_date; u.registration_date;
CREATE MATERIALIZED VIEW ybirk.mv_course_completion_rate CREATE MATERIALIZED VIEW ybirk.mv_course_completion_rate ON CLUSTER clickhouse_cluster
TO ybirk.course_completion_rate TO ybirk.course_completion_rate
AS AS
SELECT SELECT
@@ -142,16 +169,16 @@ SELECT
c.title AS course_title, c.title AS course_title,
countDistinct(l.lesson_id) AS lessons_in_course, countDistinct(l.lesson_id) AS lessons_in_course,
countDistinct(flv.lesson_id) AS lessons_viewed, countDistinct(flv.lesson_id) AS lessons_viewed,
if(countDistinct(l.lesson_id) = 0,0, if(countDistinct(l.lesson_id) = 0, 0,
round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id),2) round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id), 2)
) AS completion_rate ) AS completion_rate
FROM ybirk.core_enrollments AS e FROM ybirk.core_enrollments AS e
JOIN ybirk.core_user AS u ON e.user_id = u.user_id JOIN ybirk.core_user AS u ON e.user_id = u.user_id
JOIN ybirk.core_course AS c ON e.course_id = c.course_id JOIN ybirk.core_course AS c ON e.course_id = c.course_id
JOIN ybirk.core_lesson AS l ON c.course_id = l.course_id JOIN ybirk.core_lesson AS l ON c.course_id = l.course_id
LEFT JOIN ybirk.core_fact_lesson_views AS flv LEFT JOIN ybirk.core_fact_lesson_views AS flv
ON flv.user_id = e.user_id ON flv.user_id = e.user_id
AND flv.course_id = e.course_id AND flv.course_id = e.course_id
GROUP BY GROUP BY
u.user_id, u.user_id,
u.name, u.name,
@@ -192,4 +219,61 @@ INSERT INTO ybirk.core_enrollments (id, user_id, course_id, enrolled_at) VALUES
(2, 1, 2, '2023-02-01'), (2, 1, 2, '2023-02-01'),
(3, 2, 1, '2023-01-20'), (3, 2, 1, '2023-01-20'),
(4, 3, 2, '2023-03-05'), (4, 3, 2, '2023-03-05'),
(5, 4, 3, '2023-04-01'); (5, 4, 3, '2023-04-01');
--------------------------------------------------
--Примеры
SELECT
lesson_title,
course_title,
total_views
FROM ybirk.mv_lesson_popularity_summary
ORDER BY total_views DESC
LIMIT 5;
SELECT
name,
email,
registered_courses_count
FROM ybirk.inactive_users_summary
WHERE registered_courses_count > 0
ORDER BY registration_date DESC;
SELECT
course_title,
AVG(completion_rate) AS avg_completion
FROM ybirk.course_completion_rate
GROUP BY course_title
ORDER BY avg_completion DESC
LIMIT 5;
-- ЗАпросы
-- самый популярный курс
SELECT
course_title,
SUM(total_views) as sum_views
FROM ybirk.mv_lesson_popularity_summary
GROUP BY course_title
ORDER BY sum_views DESC;
-- Кто застрял
SELECT
user_name,
course_title,
completion_rate
FROM ybirk.mv_course_completion_rate
WHERE completion_rate > 0 AND completion_rate < 1.0;
-- тест3
SELECT
t1.user_name,
t1.course_title,
t2.email
FROM ybirk.mv_course_completion_rate AS t1
JOIN ybirk.mv_inactive_users_summary AS t2
ON t1.user_id = t2.user_id
WHERE t1.completion_rate = 1.0;