finish pet-project clickhouse

mdrozdov/clickhouse/stage2.sql

@@ -16,21 +16,27 @@ ENGINE = ReplicatedMergeTree(
 )
 ORDER BY (lesson_id, course_id);
 
+-- здесь тригер таблица dim_lesson, хотя это не совсем правильно, но мы и состояние не храним
+-- по сути таблица тригер должна быть fact_lesson_views, но right join будет плохо
+-- а если не делать right join, то можем потерять логику, потому что left join от fact_lesson_view
+-- логически теряет данные, если урок не был просмотрен
+-- но оно хотя бы работает
 create materialized view mdrozdov.lesson_popularity_summary 
+on CLUSTER clickhouse_cluster
 to mdrozdov.dm_lesson_popularity_summary as
 select 
-	dl.lesson_id 					as lesson_id,
+	dl.lesson_id as lesson_id,
-	dl.title 						as lesson_title,
+	dl.title as lesson_title,
-	dl.course_id 					as course_id,
+	dl.course_id as course_id,
-	dc.title 						as 	course_title,
+	dc.title as course_title,
-	count(flv.lesson_id) 			as total_views,
+	count(flv.lesson_id) as total_views,
-	count(distinct flv.user_id) 	as unique_users,
+	count(distinct flv.user_id) as unique_users,
-	min(flv.viewed_at) 				as first_view,
+	min(flv.viewed_at) as first_view,
-	max(flv.viewed_at) 				as last_view
+	max(flv.viewed_at) as last_view
-from mdrozdov.core_fact_lesson_views flv
+from mdrozdov.core_dim_lesson dl
-right join mdrozdov.core_dim_course dc 
+left join mdrozdov.core_dim_course dc 
-    on flv.course_id = dc.course_id
+    on dl.course_id = dc.course_id
-left join mdrozdov.core_dim_lesson dl 
+left join mdrozdov.core_fact_lesson_views flv 
     on dl.lesson_id = flv.lesson_id
 group by dl.lesson_id, dl.title, dl.course_id, dc.title
 
@@ -50,23 +56,26 @@ ENGINE = ReplicatedMergeTree(
 )
 ORDER BY user_id
 
+-- здесь тригер таблица dim_user
+-- но почему-то при вставке в dim_user новые строки в таргет таблицу не добавляются
 create materialized view mdrozdov.inactive_users_summary
-    to mdrozdov.dm_inactive_users_summary
+on CLUSTER clickhouse_cluster
-    as
+to mdrozdov.dm_inactive_users_summary
-	select
+as
-		du.user_id 				as user_id,
+select
-		du.name 				as name,
+	du.user_id as user_id,
-		du.email 				as email,
+	du.name as name,
-		du.age 					as age,
+	du.email as email,
-		count(e.course_id) 		as registered_courses_count,
+	du.age as age,
-		du.registration_date 	as registration_date
+	count(e.course_id) as registered_courses_count,
-	from mdrozdov.core_fact_lesson_views flv
+	du.registration_date as registration_date
-	right join mdrozdov.core_dim_user du 
+from mdrozdov.core_dim_user du
-		on du.user_id = flv.user_id
+left join mdrozdov.core_enrollments e 
-	left join mdrozdov.core_enrollments e 
+	on du.user_id = e.user_id
-		on du.user_id = e.user_id
+left join mdrozdov.core_fact_lesson_views flv 
-	where flv.user_id is null
+	on du.user_id = flv.user_id
-	group by du.user_id, du.name, du.email, du.age, du.registration_date
+where flv.user_id is null
+group by du.user_id, du.name, du.email, du.age, du.registration_date
 
 -- course_completion_rate
 create table mdrozdov.dm_course_completion_rate
@@ -85,42 +94,44 @@ ENGINE = ReplicatedMergeTree(
 )
 ORDER BY (user_id, course_id)
 
+-- здесь тригер таблица core_enrollments и тут все четко по логике и в работе
 create materialized view mdrozdov.course_completion_rate
-    to mdrozdov.dm_course_completion_rate as
+on CLUSTER clickhouse_cluster
-	with total_lessons_per_course as(
+to mdrozdov.dm_course_completion_rate as
-		select
+with total_lessons_per_course as(
-			course_id,
-			count(*) as lessons_in_course
-		from mdrozdov.core_dim_lesson
-		group by course_id
-	), lessons_viewed as (
-		select 
-			user_id,
-			course_id,
-			count(*) as lessons_viewed
-		from mdrozdov.core_fact_lesson_views flv
-		group by user_id, course_id
-	)
 	select
-		du.user_id                      as user_id,
+		course_id,
-		du.name                         as user_name,
+		count(*) as lessons_in_course
-		dc.course_id                    as course_id,
+	from mdrozdov.core_dim_lesson
-		dc.title                        as course_title,
+	group by course_id
-		tlpc.lessons_in_course          as lessons_in_course,
+), lessons_viewed as (
-		coalesce(lv.lessons_viewed, 0)  as lessons_viewed,
+	select 
-		round(
+		user_id,
-			coalesce(
+		course_id,
-				coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0
+		count(*) as lessons_viewed
-				), 2) as completion_rate
+	from mdrozdov.core_fact_lesson_views flv
-	from mdrozdov.core_enrollments e
+	group by user_id, course_id
-	join mdrozdov.core_dim_user du 
+)
-		on e.user_id = du.user_id
+select 
-	join total_lessons_per_course tlpc
+	du.user_id                      as user_id,
-		on tlpc.course_id = e.course_id
+	du.name                         as user_name,
-	join mdrozdov.core_dim_course dc
+	dc.course_id                    as course_id,
-		on e.course_id = dc.course_id
+	dc.title                        as course_title,
-	left join lessons_viewed lv
+	tlpc.lessons_in_course          as lessons_in_course,
-		on du.user_id = lv.user_id and dc.course_id = lv.course_id
+	coalesce(lv.lessons_viewed, 0)  as lessons_viewed,
+	round(
+		coalesce(
+			coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0
+			), 2) as completion_rate
+from mdrozdov.core_enrollments e
+join mdrozdov.core_dim_user du 
+	on e.user_id = du.user_id
+join total_lessons_per_course tlpc
+	on tlpc.course_id = e.course_id
+join mdrozdov.core_dim_course dc
+	on e.course_id = dc.course_id
+left join lessons_viewed lv
+	on du.user_id = lv.user_id and dc.course_id = lv.course_id
 
 --данные из-за несовпадения названий столбцов в импорте/экспорте пришлось удалять, короче проще вручную
 -- их добавить
@@ -132,4 +143,10 @@ insert into mdrozdov.core_fact_lesson_views(user_id, lesson_id, course_id, viewe
 (4,	4,	3,	'2023-04-02 10:00:00'),
 (4,	5,	3,	'2023-04-03 10:00:00')
 
--- данные в mdrozdov.core_enrollments загружал через импорт
+insert into enrollments (user_id, course_id, enrolled_at) values
+(1,	1,	'2023-01-15'),
+(1,	2,	'2023-02-01'),
+(2,	1,	'2023-01-20'),
+(3,	2,	'2023-03-05'),
+(4,	3,	'2023-04-01')
+

mdrozdov/clickhouse/stage3-4.sql

@@ -0,0 +1,45 @@
+-- Топ-5 уроков с наименьшим количеством уникальных пользователей
+SELECT 
+    lesson_title,
+    course_title,
+    unique_users
+FROM lesson_popularity_summary
+ORDER BY unique_users ASC
+LIMIT 5;
+
+-- Пользователи без зарегистрированных курсов
+SELECT 
+    name,
+    email,
+    registration_date
+FROM inactive_users_summary
+WHERE registered_courses_count = 0
+ORDER BY registration_date ASC;
+
+-- Список курсов с минимальной средней степенью завершения
+SELECT 
+    course_title,
+    AVG(completion_rate) AS avg_completion
+FROM course_completion_rate
+GROUP BY course_title
+ORDER BY avg_completion ASC
+LIMIT 5;
+
+-- Мини отчет:
+/*
+Таблица lesson_popularity_summary
+    Выбранные поля для проверки:
+        total_views - агрегат count
+        first_view - агрегат min
+    оба совпадают
+Таблица inactive_users_summary
+    Выбранные поля для проверки:
+        registered_courses_count - агрегат count
+    совпадает
+Таблица course_completion_rate
+    Выбранные поля для проверки:
+        completion_rate - агрегат count
+    совпадает
+Итог: в моем случае все значения совпали, возможно если бы это было не через простой MergeTree
+все было бы хуже
+*/