WIP: pet-project clickhouse MVs

mdrozdov/clickhouse/stage2.sql

@@ -0,0 +1,123 @@
+--lesson_popularity_summary data mart
+create table mdrozdov.dm_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster(
+    lesson_id               UInt32,
+    lesson_title            String,
+    course_id               UInt32,
+    course_title            String,
+    total_views             AggregateFunction(count, UInt64),
+    unique_users            AggregateFunction(uniqExact, Int32),
+    first_view              AggregateFunction(min, DateTime),
+    last_view               AggregateFunction(max, DateTime)
+)
+ENGINE = ReplicatedAggregatingMergeTree(
+'/clickhouse/tables/{shard}/mdrozdov/dm_lesson_popularity_summary',
+'{replica}'
+)
+ORDER BY (lesson_id, course_id);
+
+create materialized view mdrozdov.lesson_popularity_summary to mdrozdov.dm_lesson_popularity_summary as
+select 
+	dl.lesson_id as lesson_id,
+	dl.title as lesson_title,
+	dl.course_id as course_id,
+	dc.title as course_title,
+	countState(flv.lesson_id) as total_views,
+	uniqExactState(flv.user_id) as unique_users,
+	minState(toDateTime(flv.viewed_at)) as first_view,
+	maxState(toDateTime(flv.viewed_at)) as last_view
+from mdrozdov.core_fact_lesson_views flv
+left join mdrozdov.core_dim_lesson dl 
+    on dl.lesson_id = flv.lesson_id
+left join mdrozdov.core_dim_course dc 
+    on flv.course_id = dc.course_id
+group by dl.lesson_id, dl.title, dl.course_id, dc.title
+
+
+--inactive_users_summary
+create table mdrozdov.dm_inactive_users_summary
+ON CLUSTER clickhouse_cluster(
+    user_id                         Int32,
+    name                            String,
+    email                           Nullable(String),
+    age                             Int32,
+    registered_courses_count        AggregateFunction(uniqExact, Int32),
+    registration_date               Date
+)
+ENGINE = ReplicatedAggregatingMergeTree(
+    '/clickhouse/tables/{shard}/mdrozdov/dm_inactive_users_summary',
+    '{replica}'
+)
+ORDER BY user_id
+
+create materialized view mdrozdov.inactive_users_summary
+    to mdrozdov.dm_inactive_users_summary
+    as
+	select
+		du.user_id as user_id,
+		du.name as name,
+		du.email as email,
+		du.age as age,
+		uniqExactState(e.course_id) as registered_courses_count,
+		du.registration_date
+	from mdrozdov.core_fact_lesson_views flv
+	right join mdrozdov.core_dim_user du 
+		on du.user_id = flv.user_id
+	left join mdrozdov.core_enrollments e 
+		on du.user_id = e.user_id
+	where flv.user_id is null
+	group by du.user_id, du.name, du.email, du.age, du.registration_date
+
+-- course_completion_rate
+create table mdrozdov.dm_course_completion_rate
+ON CLUSTER clickhouse_cluster(
+    user_id                     Int32,
+    user_name                   String,
+    course_id                   Int32,
+    course_title                String,
+    lessons_in_course           AggregateFunction(count, UInt64),
+    lessons_viewed              AggregateFunction(count, UInt64),
+    completion_rate             Decimal(4, 3)
+)
+ENGINE = ReplicatedAggregatingMergeTree(
+    '/clickhouse/tables/{shard}/mdrozdov/dm_course_completion_rate',
+    '{replica}'
+)
+ORDER BY (user_id, course_id)
+
+create materialized view mdrozdov.course_completion_rate
+    to mdrozdov.dm_course_completion_rate as
+	with total_lessons_per_course as(
+		select
+			course_id,
+			count(*) as lessons_in_course
+		from core.dim_lesson
+		group by course_id
+	), lessons_viewed as (
+		select 
+			user_id,
+			course_id,
+			count(*) as lessons_viewed
+		from fact_lesson_views flv
+		group by user_id, course_id
+	)
+	select 
+		du.user_id                      as user_id,
+		du.name                         as user_name,
+		dc.course_id                    as course_id,
+		dc.title                        as course_title,
+		tlpc.lessons_in_course          as lessons_in_course,
+		coalesce(lv.lessons_viewed, 0)  as lessons_viewed,
+		round(
+			coalesce(
+				coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0
+				), 2) as completion_rate
+	from mdrozdov.core_enrollments e
+	join mdrozdov.core_dim_user du 
+		on e.user_id = du.user_id
+	join total_lessons_per_course tlpc
+		on tlpc.course_id = e.course_id
+	join mdrozdov.core_dim_course dc
+		on e.course_id = dc.course_id
+	left join lessons_viewed lv
+		on du.user_id = lv.user_id and dc.course_id = lv.course_id