dmitry_medvedev/homework
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request 'OS-319' (#76) from OS-319 into main

Browse Source 
Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/76
Reviewed-by: aleksey bogatyrev <abogatyrev@example.com>
This commit is contained in:
aleksey bogatyrev
2026-02-09 16:58:15 +04:00
parent 2d94499d1f 84fd29ea9e
commit 53d802bb89
3 changed files with 293 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mdrozdov/clickhouse/stage1.sql
+96
View File
@@ -0,0 +1,96 @@
-- удалить таблицу если раньше создавали
DROP TABLE IF EXISTS mdrozdov.core_fact_lesson_views
ON CLUSTER clickhouse_cluster;
-- таблицы измерений
-- user dimension
CREATE TABLE mdrozdov.core_dim_user
ON CLUSTER clickhouse_cluster
(
user_id Int32,
name String,
age Int32,
email String,
registration_date Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/core_dim_user',
'{replica}'
)
ORDER BY user_id;
-- course dimension
CREATE TABLE mdrozdov.core_dim_course
ON CLUSTER clickhouse_cluster
(
course_id Int32,
title String,
category String
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/core_dim_course',
'{replica}'
)
ORDER BY course_id;
-- lesson dimension
CREATE TABLE mdrozdov.core_dim_lesson
ON CLUSTER clickhouse_cluster
(
lesson_id Int32,
title String,
duration_min Int32,
course_id Int32,
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/core_dim_lesson2',
'{replica}'
)
ORDER BY (course_id, lesson_id);
-- date dimension
CREATE TABLE mdrozdov.core_dim_date
ON CLUSTER clickhouse_cluster
(
date DATE,
year Int32,
month Int32,
day Int32,
weekday String
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/core_dim_date',
'{replica}'
)
ORDER BY date;
-- таблицы фактов
-- core_enrollments
CREATE TABLE mdrozdov.core_enrollments
ON CLUSTER clickhouse_cluster
(
user_id Int32,
course_id Int32,
enrolled_at DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/core_enrollments',
'{replica}'
)
ORDER BY course_id;
-- core_fact_lesson_views
CREATE TABLE mdrozdov.core_fact_lesson_views
ON CLUSTER clickhouse_cluster
(
user_id Int32,
lesson_id Int32,
course_id Int32,
viewed_at DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/core_fact_lesson_views',
'{replica}'
)
ORDER BY course_id;
mdrozdov/clickhouse/stage2.sql
+152
View File
@@ -0,0 +1,152 @@
--lesson_popularity_summary data mart
create table mdrozdov.dm_lesson_popularity_summary
ON CLUSTER clickhouse_cluster(
lesson_id Int32,
lesson_title String,
course_id Int32,
course_title String,
total_views Int32,
unique_users Int32,
first_view DateTime64(3),
last_view DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/dm_lesson_popularity_summary1',
'{replica}'
)
ORDER BY (lesson_id, course_id);
-- здесь тригер таблица dim_lesson, хотя это не совсем правильно, но мы и состояние не храним
-- по сути таблица тригер должна быть fact_lesson_views, но right join будет плохо
-- а если не делать right join, то можем потерять логику, потому что left join от fact_lesson_view
-- логически теряет данные, если урок не был просмотрен
-- но оно хотя бы работает
create materialized view mdrozdov.lesson_popularity_summary
on CLUSTER clickhouse_cluster
to mdrozdov.dm_lesson_popularity_summary as
select
dl.lesson_id as lesson_id,
dl.title as lesson_title,
dl.course_id as course_id,
dc.title as course_title,
count(flv.lesson_id) as total_views,
count(distinct flv.user_id) as unique_users,
min(flv.viewed_at) as first_view,
max(flv.viewed_at) as last_view
from mdrozdov.core_dim_lesson dl
left join mdrozdov.core_dim_course dc
on dl.course_id = dc.course_id
left join mdrozdov.core_fact_lesson_views flv
on dl.lesson_id = flv.lesson_id
group by dl.lesson_id, dl.title, dl.course_id, dc.title
--inactive_users_summary
create table mdrozdov.dm_inactive_users_summary
ON CLUSTER clickhouse_cluster(
user_id Int32,
name String,
email String,
age Int32,
registered_courses_count Int32,
registration_date Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/dm_inactive_users_summary1',
'{replica}'
)
ORDER BY user_id
-- здесь тригер таблица dim_user
-- но почему-то при вставке в dim_user новые строки в таргет таблицу не добавляются
create materialized view mdrozdov.inactive_users_summary
on CLUSTER clickhouse_cluster
to mdrozdov.dm_inactive_users_summary
as
select
du.user_id as user_id,
du.name as name,
du.email as email,
du.age as age,
count(e.course_id) as registered_courses_count,
du.registration_date as registration_date
from mdrozdov.core_dim_user du
left join mdrozdov.core_enrollments e
on du.user_id = e.user_id
left join mdrozdov.core_fact_lesson_views flv
on du.user_id = flv.user_id
where flv.user_id is null
group by du.user_id, du.name, du.email, du.age, du.registration_date
-- course_completion_rate
create table mdrozdov.dm_course_completion_rate
ON CLUSTER clickhouse_cluster(
user_id Int32,
user_name String,
course_id Int32,
course_title String,
lessons_in_course Int32,
lessons_viewed Int32,
completion_rate Decimal(4, 3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/mdrozdov/dm_course_completion_rate',
'{replica}'
)
ORDER BY (user_id, course_id)
-- здесь тригер таблица core_enrollments и тут все четко по логике и в работе
create materialized view mdrozdov.course_completion_rate
on CLUSTER clickhouse_cluster
to mdrozdov.dm_course_completion_rate as
with total_lessons_per_course as(
select
course_id,
count(*) as lessons_in_course
from mdrozdov.core_dim_lesson
group by course_id
), lessons_viewed as (
select
user_id,
course_id,
count(*) as lessons_viewed
from mdrozdov.core_fact_lesson_views flv
group by user_id, course_id
)
select
du.user_id as user_id,
du.name as user_name,
dc.course_id as course_id,
dc.title as course_title,
tlpc.lessons_in_course as lessons_in_course,
coalesce(lv.lessons_viewed, 0) as lessons_viewed,
round(
coalesce(
coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0
), 2) as completion_rate
from mdrozdov.core_enrollments e
join mdrozdov.core_dim_user du
on e.user_id = du.user_id
join total_lessons_per_course tlpc
on tlpc.course_id = e.course_id
join mdrozdov.core_dim_course dc
on e.course_id = dc.course_id
left join lessons_viewed lv
on du.user_id = lv.user_id and dc.course_id = lv.course_id
--данные из-за несовпадения названий столбцов в импорте/экспорте пришлось удалять, короче проще вручную
-- их добавить
insert into mdrozdov.core_fact_lesson_views(user_id, lesson_id, course_id, viewed_at) values
(1, 1, 1, '2023-01-16 10:00:00'),
(1, 2, 1, '2023-01-16 10:15:00'),
(2, 1, 1, '2023-01-21 09:00:00'),
(3, 3, 2, '2023-03-06 12:00:00'),
(4, 4, 3, '2023-04-02 10:00:00'),
(4, 5, 3, '2023-04-03 10:00:00')
insert into enrollments (user_id, course_id, enrolled_at) values
(1, 1, '2023-01-15'),
(1, 2, '2023-02-01'),
(2, 1, '2023-01-20'),
(3, 2, '2023-03-05'),
(4, 3, '2023-04-01')
mdrozdov/clickhouse/stage3-4.sql
+45
View File
@@ -0,0 +1,45 @@
-- Топ-5 уроков с наименьшим количеством уникальных пользователей
SELECT
lesson_title,
course_title,
unique_users
FROM lesson_popularity_summary
ORDER BY unique_users ASC
LIMIT 5;
-- Пользователи без зарегистрированных курсов
SELECT
name,
email,
registration_date
FROM inactive_users_summary
WHERE registered_courses_count = 0
ORDER BY registration_date ASC;
-- Список курсов с минимальной средней степенью завершения
SELECT
course_title,
AVG(completion_rate) AS avg_completion
FROM course_completion_rate
GROUP BY course_title
ORDER BY avg_completion ASC
LIMIT 5;
-- Мини отчет:
/*
Таблица lesson_popularity_summary
Выбранные поля для проверки:
total_views - агрегат count
first_view - агрегат min
оба совпадают
Таблица inactive_users_summary
Выбранные поля для проверки:
registered_courses_count - агрегат count
совпадает
Таблица course_completion_rate
Выбранные поля для проверки:
completion_rate - агрегат count
совпадает
Итог: в моем случае все значения совпали, возможно если бы это было не через простой MergeTree
все было бы хуже
*/