homework/ygurbanberdiev/clickhouse/Pet_Project/Pet_project_clickhouse.sql

-- Clickhouse. Пет-проект: Витринный слой
CREATE TABLE core_dim_user 
ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    name String,
    age UInt8,
    email String,
    registration_date Date
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_user',
    '{replica}'
)
ORDER BY user_id;


CREATE TABLE core_dim_course 
ON CLUSTER clickhouse_cluster (
    course_id UInt32,
    title String,
    category String
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_course',
    '{replica}'
)
ORDER BY course_id;


CREATE TABLE core_dim_lesson 
ON CLUSTER clickhouse_cluster (
    lesson_id UInt32,
    title String,
    duration_min UInt8,
    course_id UInt32
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_lesson',
    '{replica}'
)
ORDER BY (course_id, lesson_id);


CREATE TABLE core_dim_date 
ON CLUSTER clickhouse_cluster (
    date Date,
    year UInt16,
    month UInt8,
    weekday String
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_date',
    '{replica}'
)
ORDER BY date;


CREATE TABLE core_fact_lesson_views 
ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    lesson_id UInt32,
    course_id UInt32,
    viewed_at DateTime64(0)
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_fact_lesson_views',
    '{replica}'
)
ORDER BY (user_id, course_id, viewed_at, lesson_id);


CREATE TABLE core_enrollments
ON CLUSTER clickhouse_cluster (
    user_id UInt32,
    course_id UInt32,
    enrolled_at DateTime64(0)
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/core_enrollments',
    '{replica}'
)
ORDER BY (user_id, course_id);



-- Этап 2. Витрина lesson_popularity_summary
CREATE TABLE lesson_popularity_summary
ON CLUSTER clickhouse_cluster (
	lesson_id UInt32,
	lesson_title String,
	course_id UInt32,
	course_title String,
	total_views UInt64,
	unique_users UInt64,
	first_view DateTime64(0),
	last_view DateTime64(0)
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/lesson_popularity_summary',
    '{replica}'
)
ORDER BY (lesson_id, course_id);

CREATE MATERIALIZED VIEW mv_lesson_popularity_summary
ON CLUSTER clickhouse_cluster
TO lesson_popularity_summary AS 
WITH agg_table AS (
SELECT
	lesson_id,
	course_id,
	count() AS total_views,
	uniqExact(user_id) AS unique_users,
	min(viewed_at) AS first_view,
	max(viewed_at) AS last_view
FROM core_fact_lesson_views
GROUP BY lesson_id, course_id
)
SELECT
	atab.lesson_id AS lesson_id,
	cdl.title AS lesson_title,
	atab.course_id AS course_id,
	cdc.title AS course_title,
	atab.total_views AS total_views,
	atab.unique_users AS unique_users,
	atab.first_view AS first_view,
	atab.last_view AS last_view
FROM agg_table AS atab
JOIN core_dim_lesson AS cdl ON atab.lesson_id = cdl.lesson_id 
JOIN core_dim_course AS cdc ON atab.course_id = cdc.course_id;


-- Этап 2. Витрина inactive_users_summary
CREATE TABLE inactive_users_summary
ON CLUSTER clickhouse_cluster (
	user_id UInt64,
	user_name String,
	email String,
	age UInt8,
	registered_courses_count UInt64,
	registration_date Date
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/inactive_users_summary',
    '{replica}'
)
ORDER BY user_id;


CREATE MATERIALIZED VIEW mv_inactive_users_summary
ON CLUSTER clickhouse_cluster
TO inactive_users_summary
AS
SELECT
    u.user_id AS user_id,
    u.name AS user_name,
    u.email AS email,
    u.age AS age,
    COALESCE(e.registered_courses_count, 0) AS registered_courses_count,
    u.registration_date AS registration_date
FROM core_dim_user AS u
LEFT JOIN
(	SELECT
		user_id,
		count() AS registered_courses_count
	FROM core_enrollments
   GROUP BY user_id
) AS e ON u.user_id = e.user_id
LEFT JOIN
(	SELECT DISTINCT
   		user_id
	FROM core_fact_lesson_views
) AS v ON u.user_id = v.user_id
WHERE v.user_id IS NULL;



-- Этап 2. Витрина course_completion_rate
CREATE TABLE course_completion_rate
ON CLUSTER clickhouse_cluster
(
    user_id UInt32,
    user_name String,
    course_id UInt32,
    course_title String,
    lessons_in_course UInt64,
    lessons_viewed UInt64,
    completion_rate Float64
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/tables/{shard}/ygurbanberdiev/course_completion_rate',
    '{replica}'
)
ORDER BY (user_id, course_id);

CREATE MATERIALIZED VIEW mv_course_completion_rate
ON CLUSTER clickhouse_cluster
TO course_completion_rate
AS
SELECT
	e.user_id AS user_id,
	u.name AS user_name,
	e.course_id AS course_id,
	c.title AS course_title,
	COALESCE(lc.lessons_in_course, 0) AS lessons_in_course,
	COALESCE(lv.lessons_viewed, 0) AS lessons_viewed,
	if(COALESCE(lc.lessons_in_course, 0) = 0,0, round(COALESCE(lv.lessons_viewed, 0) * 1.0 / lc.lessons_in_course, 2)) AS completion_rate
FROM core_enrollments AS e
LEFT JOIN core_dim_user AS u ON e.user_id = u.user_id
LEFT JOIN core_dim_course AS c ON e.course_id = c.course_id
LEFT JOIN
(	SELECT
		course_id,
		count() AS lessons_in_course
	FROM core_dim_lesson
	GROUP BY course_id
) AS lc ON e.course_id = lc.course_id
LEFT JOIN
(	SELECT
		user_id,
		course_id,
		uniqExact(lesson_id) AS lessons_viewed
	FROM core_fact_lesson_views
	GROUP BY user_id, course_id
) AS lv ON e.user_id = lv.user_id
   AND e.course_id = lv.course_id;


	
-- Этап 3. Уроки которые почти никто не смотрел
SELECT
    lesson_title,
    course_title,
    total_views,
    unique_users
FROM lesson_popularity_summary
WHERE total_views <= 1
ORDER BY total_views ASC;

-- Неактивные пользователи без записей на курсы
SELECT
    user_name,
    email,
    age,
    registration_date
FROM inactive_users_summary
WHERE registered_courses_count = 0
ORDER BY registration_date DESC;

-- Пользователи, полностью завершившие курс
SELECT
    user_name,
    course_title,
    lessons_in_course,
    lessons_viewed,
    completion_rate
FROM course_completion_rate
WHERE completion_rate = 1
ORDER BY course_title, user_name;

/* Отчет 1: При сравнении витрины lesson_popularity_summary в ClickHouse и Greenplum результаты совпали только частично
Совпали показатели:
SUM(total_views) = 58
MAX(total_views) = 9

Не совпали показатели:
AVG(total_views)
MIN(total_views) 

В Clickhouse:
AVG(total_views) = 3.2222
MIN(total_views) = 0
В Greenplum: 
AVG(total_views) = 3.8666
MIN(total_views) = 1

Причина расхождения: в ClickHouse в витрине есть дополнительные строки с total_views = 0, а в Greenplum таких строк нет. Из-за этих нулевых строк минимальное значение в ClickHouse стало 0, а среднее значение уменьшилось 


Отчет 2: При проверке витрины inactive_users_summary результаты в ClickHouse и Greenplum совпали полностью.
В обеих системах получены одинаковые значения:
inactive_users_count = 9
sum_registered_courses = 17
avg_registered_courses = 1.8888888889
min_registered_courses = 1
max_registered_courses = 3

*/