миграция таблиц на ReplicatedMergeTree с поддержкой кластера
добавил примеры для проверки
This commit is contained in:
Birk
@@ -1,94 +1,121 @@
|
||||
CREATE TABLE ybirk.core_user (
|
||||
CREATE TABLE ybirk.core_user ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id Int32,
|
||||
name String,
|
||||
age Int32,
|
||||
email String,
|
||||
registration_date Date
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/core_user',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY user_id;
|
||||
|
||||
CREATE TABLE ybirk.core_course (
|
||||
CREATE TABLE ybirk.core_course ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
course_id Int32,
|
||||
title String,
|
||||
category String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/core_course',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY course_id;
|
||||
|
||||
CREATE TABLE ybirk.core_lesson (
|
||||
CREATE TABLE ybirk.core_lesson ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
lesson_id Int32,
|
||||
title String,
|
||||
duration_min Int32,
|
||||
course_id Int32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/core_lesson',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY lesson_id;
|
||||
|
||||
CREATE TABLE ybirk.core_fact_lesson_views (
|
||||
CREATE TABLE ybirk.core_fact_lesson_views ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id Int32,
|
||||
lesson_id Int32,
|
||||
course_id Int32,
|
||||
viewed_at DateTime64(3)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/core_fact_lesson_views',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY user_id;
|
||||
|
||||
CREATE TABLE ybirk.core_enrollments (
|
||||
CREATE TABLE ybirk.core_enrollments ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
id Int32,
|
||||
user_id Int32,
|
||||
course_id Int32,
|
||||
enrolled_at Date
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/core_enrollments',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY user_id;
|
||||
|
||||
----------------------------------
|
||||
--------------------------------
|
||||
|
||||
CREATE TABLE ybirk.inactive_users_summary
|
||||
CREATE TABLE ybirk.inactive_users_summary ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id UInt32,
|
||||
name String,
|
||||
email String,
|
||||
age UInt8,
|
||||
registered_courses_count UInt32,
|
||||
registration_date DateTime64(3, 'UTC')
|
||||
user_id UInt32,
|
||||
name String,
|
||||
email String,
|
||||
age UInt8,
|
||||
registered_courses_count UInt32,
|
||||
registration_date DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = AggregatingMergeTree
|
||||
ORDER BY (user_id);
|
||||
ENGINE = ReplicatedAggregatingMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/inactive_users_summary',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY user_id;
|
||||
|
||||
|
||||
CREATE TABLE ybirk.lesson_popularity_summary
|
||||
CREATE TABLE ybirk.lesson_popularity_summary ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
lesson_id UInt32,
|
||||
lesson_title String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
total_views UInt32,
|
||||
unique_users UInt32,
|
||||
first_view DateTime64(3, 'UTC'),
|
||||
last_view DateTime64(3, 'UTC')
|
||||
lesson_id UInt32,
|
||||
lesson_title String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
total_views UInt32,
|
||||
unique_users UInt32,
|
||||
first_view DateTime64(3, 'UTC'),
|
||||
last_view DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = ReplicatedAggregatingMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/lesson_popularity_summary',
|
||||
'{replica}'
|
||||
)
|
||||
ENGINE = AggregatingMergeTree
|
||||
ORDER BY (lesson_id, course_id);
|
||||
|
||||
|
||||
CREATE TABLE ybirk.course_completion_rate
|
||||
CREATE TABLE ybirk.course_completion_rate ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id UInt32,
|
||||
user_name String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
user_id UInt32,
|
||||
user_name String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
lessons_in_course UInt32,
|
||||
lessons_viewed UInt32,
|
||||
completion_rate Float32
|
||||
lessons_viewed UInt32,
|
||||
completion_rate Float32
|
||||
)
|
||||
ENGINE = ReplicatedAggregatingMergeTree(
|
||||
'/clickhouse/tables/{shard}/ybirk/course_completion_rate',
|
||||
'{replica}'
|
||||
)
|
||||
ENGINE = AggregatingMergeTree
|
||||
ORDER BY (user_id, course_id);
|
||||
|
||||
-------------------------------------------
|
||||
--------------------------------------
|
||||
|
||||
CREATE MATERIALIZED VIEW ybirk.mv_lesson_popularity_summary
|
||||
CREATE MATERIALIZED VIEW ybirk.mv_lesson_popularity_summary ON CLUSTER clickhouse_cluster
|
||||
TO ybirk.lesson_popularity_summary
|
||||
AS
|
||||
SELECT
|
||||
@@ -101,15 +128,15 @@ SELECT
|
||||
min(flv.viewed_at) AS first_view,
|
||||
max(flv.viewed_at) AS last_view
|
||||
FROM ybirk.core_fact_lesson_views AS flv
|
||||
JOIN ybirk.core_lesson AS l ON flv.lesson_id = l.lesson_id
|
||||
JOIN ybirk.core_course AS c ON flv.course_id = c.course_id
|
||||
JOIN ybirk.core_lesson AS l ON flv.lesson_id = l.lesson_id
|
||||
JOIN ybirk.core_course AS c ON flv.course_id = c.course_id
|
||||
GROUP BY
|
||||
flv.lesson_id,
|
||||
l.title,
|
||||
flv.course_id,
|
||||
c.title;
|
||||
|
||||
CREATE MATERIALIZED VIEW ybirk.mv_inactive_users_summary
|
||||
CREATE MATERIALIZED VIEW ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster
|
||||
TO ybirk.inactive_users_summary
|
||||
AS
|
||||
SELECT
|
||||
@@ -132,7 +159,7 @@ GROUP BY
|
||||
u.age,
|
||||
u.registration_date;
|
||||
|
||||
CREATE MATERIALIZED VIEW ybirk.mv_course_completion_rate
|
||||
CREATE MATERIALIZED VIEW ybirk.mv_course_completion_rate ON CLUSTER clickhouse_cluster
|
||||
TO ybirk.course_completion_rate
|
||||
AS
|
||||
SELECT
|
||||
@@ -142,16 +169,16 @@ SELECT
|
||||
c.title AS course_title,
|
||||
countDistinct(l.lesson_id) AS lessons_in_course,
|
||||
countDistinct(flv.lesson_id) AS lessons_viewed,
|
||||
if(countDistinct(l.lesson_id) = 0,0,
|
||||
round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id),2)
|
||||
if(countDistinct(l.lesson_id) = 0, 0,
|
||||
round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id), 2)
|
||||
) AS completion_rate
|
||||
FROM ybirk.core_enrollments AS e
|
||||
JOIN ybirk.core_user AS u ON e.user_id = u.user_id
|
||||
JOIN ybirk.core_course AS c ON e.course_id = c.course_id
|
||||
JOIN ybirk.core_lesson AS l ON c.course_id = l.course_id
|
||||
JOIN ybirk.core_user AS u ON e.user_id = u.user_id
|
||||
JOIN ybirk.core_course AS c ON e.course_id = c.course_id
|
||||
JOIN ybirk.core_lesson AS l ON c.course_id = l.course_id
|
||||
LEFT JOIN ybirk.core_fact_lesson_views AS flv
|
||||
ON flv.user_id = e.user_id
|
||||
AND flv.course_id = e.course_id
|
||||
ON flv.user_id = e.user_id
|
||||
AND flv.course_id = e.course_id
|
||||
GROUP BY
|
||||
u.user_id,
|
||||
u.name,
|
||||
@@ -192,4 +219,61 @@ INSERT INTO ybirk.core_enrollments (id, user_id, course_id, enrolled_at) VALUES
|
||||
(2, 1, 2, '2023-02-01'),
|
||||
(3, 2, 1, '2023-01-20'),
|
||||
(4, 3, 2, '2023-03-05'),
|
||||
(5, 4, 3, '2023-04-01');
|
||||
(5, 4, 3, '2023-04-01');
|
||||
|
||||
--------------------------------------------------
|
||||
|
||||
--Примеры
|
||||
|
||||
SELECT
|
||||
lesson_title,
|
||||
course_title,
|
||||
total_views
|
||||
FROM ybirk.mv_lesson_popularity_summary
|
||||
ORDER BY total_views DESC
|
||||
LIMIT 5;
|
||||
|
||||
SELECT
|
||||
name,
|
||||
email,
|
||||
registered_courses_count
|
||||
FROM ybirk.inactive_users_summary
|
||||
WHERE registered_courses_count > 0
|
||||
ORDER BY registration_date DESC;
|
||||
|
||||
SELECT
|
||||
course_title,
|
||||
AVG(completion_rate) AS avg_completion
|
||||
FROM ybirk.course_completion_rate
|
||||
GROUP BY course_title
|
||||
ORDER BY avg_completion DESC
|
||||
LIMIT 5;
|
||||
|
||||
-- ЗАпросы
|
||||
|
||||
-- самый популярный курс
|
||||
SELECT
|
||||
course_title,
|
||||
SUM(total_views) as sum_views
|
||||
FROM ybirk.mv_lesson_popularity_summary
|
||||
GROUP BY course_title
|
||||
ORDER BY sum_views DESC;
|
||||
|
||||
|
||||
-- Кто застрял
|
||||
SELECT
|
||||
user_name,
|
||||
course_title,
|
||||
completion_rate
|
||||
FROM ybirk.mv_course_completion_rate
|
||||
WHERE completion_rate > 0 AND completion_rate < 1.0;
|
||||
|
||||
-- тест3
|
||||
SELECT
|
||||
t1.user_name,
|
||||
t1.course_title,
|
||||
t2.email
|
||||
FROM ybirk.mv_course_completion_rate AS t1
|
||||
JOIN ybirk.mv_inactive_users_summary AS t2
|
||||
ON t1.user_id = t2.user_id
|
||||
WHERE t1.completion_rate = 1.0;
|
||||
Reference in New Issue
Block a user