homework, pet project DE Dmitry_Medvedev #1
@@ -0,0 +1,287 @@
|
||||
-- Clickhouse. Пет-проект: Витринный слой
|
||||
CREATE TABLE core_dim_user
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
user_id UInt32,
|
||||
name String,
|
||||
age UInt8,
|
||||
email String,
|
||||
registration_date Date
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_user',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY user_id;
|
||||
|
||||
|
||||
CREATE TABLE core_dim_course
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
course_id UInt32,
|
||||
title String,
|
||||
category String
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_course',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY course_id;
|
||||
|
||||
|
||||
CREATE TABLE core_dim_lesson
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
lesson_id UInt32,
|
||||
title String,
|
||||
duration_min UInt8,
|
||||
course_id UInt32
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_lesson',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (course_id, lesson_id);
|
||||
|
||||
|
||||
CREATE TABLE core_dim_date
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
date Date,
|
||||
year UInt16,
|
||||
month UInt8,
|
||||
weekday String
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/core_dim_date',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY date;
|
||||
|
||||
|
||||
CREATE TABLE core_fact_lesson_views
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
user_id UInt32,
|
||||
lesson_id UInt32,
|
||||
course_id UInt32,
|
||||
viewed_at DateTime64(0)
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/core_fact_lesson_views',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (user_id, course_id, viewed_at, lesson_id);
|
||||
|
||||
|
||||
CREATE TABLE core_enrollments
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
user_id UInt32,
|
||||
course_id UInt32,
|
||||
enrolled_at DateTime64(0)
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/core_enrollments',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (user_id, course_id);
|
||||
|
||||
|
||||
|
||||
-- Этап 2. Витрина lesson_popularity_summary
|
||||
CREATE TABLE lesson_popularity_summary
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
lesson_id UInt32,
|
||||
lesson_title String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
total_views UInt64,
|
||||
unique_users UInt64,
|
||||
first_view DateTime64(0),
|
||||
last_view DateTime64(0)
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/lesson_popularity_summary',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (lesson_id, course_id);
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_lesson_popularity_summary
|
||||
ON CLUSTER clickhouse_cluster
|
||||
TO lesson_popularity_summary AS
|
||||
WITH agg_table AS (
|
||||
SELECT
|
||||
lesson_id,
|
||||
course_id,
|
||||
count() AS total_views,
|
||||
uniqExact(user_id) AS unique_users,
|
||||
min(viewed_at) AS first_view,
|
||||
max(viewed_at) AS last_view
|
||||
FROM core_fact_lesson_views
|
||||
GROUP BY lesson_id, course_id
|
||||
)
|
||||
SELECT
|
||||
atab.lesson_id AS lesson_id,
|
||||
cdl.title AS lesson_title,
|
||||
atab.course_id AS course_id,
|
||||
cdc.title AS course_title,
|
||||
atab.total_views AS total_views,
|
||||
atab.unique_users AS unique_users,
|
||||
atab.first_view AS first_view,
|
||||
atab.last_view AS last_view
|
||||
FROM agg_table AS atab
|
||||
JOIN core_dim_lesson AS cdl ON atab.lesson_id = cdl.lesson_id
|
||||
JOIN core_dim_course AS cdc ON atab.course_id = cdc.course_id;
|
||||
|
||||
|
||||
-- Этап 2. Витрина inactive_users_summary
|
||||
CREATE TABLE inactive_users_summary
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
user_id UInt64,
|
||||
user_name String,
|
||||
email String,
|
||||
age UInt8,
|
||||
registered_courses_count UInt64,
|
||||
registration_date Date
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/inactive_users_summary',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY user_id;
|
||||
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_inactive_users_summary
|
||||
ON CLUSTER clickhouse_cluster
|
||||
TO inactive_users_summary
|
||||
AS
|
||||
SELECT
|
||||
u.user_id AS user_id,
|
||||
u.name AS user_name,
|
||||
u.email AS email,
|
||||
u.age AS age,
|
||||
COALESCE(e.registered_courses_count, 0) AS registered_courses_count,
|
||||
u.registration_date AS registration_date
|
||||
FROM core_dim_user AS u
|
||||
LEFT JOIN
|
||||
( SELECT
|
||||
user_id,
|
||||
count() AS registered_courses_count
|
||||
FROM core_enrollments
|
||||
GROUP BY user_id
|
||||
) AS e ON u.user_id = e.user_id
|
||||
LEFT JOIN
|
||||
( SELECT DISTINCT
|
||||
user_id
|
||||
FROM core_fact_lesson_views
|
||||
) AS v ON u.user_id = v.user_id
|
||||
WHERE v.user_id IS NULL;
|
||||
|
||||
|
||||
|
||||
-- Этап 2. Витрина course_completion_rate
|
||||
CREATE TABLE course_completion_rate
|
||||
ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id UInt32,
|
||||
user_name String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
lessons_in_course UInt64,
|
||||
lessons_viewed UInt64,
|
||||
completion_rate Float64
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/ygurbanberdiev/course_completion_rate',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (user_id, course_id);
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_course_completion_rate
|
||||
ON CLUSTER clickhouse_cluster
|
||||
TO course_completion_rate
|
||||
AS
|
||||
SELECT
|
||||
e.user_id AS user_id,
|
||||
u.name AS user_name,
|
||||
e.course_id AS course_id,
|
||||
c.title AS course_title,
|
||||
COALESCE(lc.lessons_in_course, 0) AS lessons_in_course,
|
||||
COALESCE(lv.lessons_viewed, 0) AS lessons_viewed,
|
||||
if(COALESCE(lc.lessons_in_course, 0) = 0,0, round(COALESCE(lv.lessons_viewed, 0) * 1.0 / lc.lessons_in_course, 2)) AS completion_rate
|
||||
FROM core_enrollments AS e
|
||||
LEFT JOIN core_dim_user AS u ON e.user_id = u.user_id
|
||||
LEFT JOIN core_dim_course AS c ON e.course_id = c.course_id
|
||||
LEFT JOIN
|
||||
( SELECT
|
||||
course_id,
|
||||
count() AS lessons_in_course
|
||||
FROM core_dim_lesson
|
||||
GROUP BY course_id
|
||||
) AS lc ON e.course_id = lc.course_id
|
||||
LEFT JOIN
|
||||
( SELECT
|
||||
user_id,
|
||||
course_id,
|
||||
uniqExact(lesson_id) AS lessons_viewed
|
||||
FROM core_fact_lesson_views
|
||||
GROUP BY user_id, course_id
|
||||
) AS lv ON e.user_id = lv.user_id
|
||||
AND e.course_id = lv.course_id;
|
||||
|
||||
|
||||
|
||||
-- Этап 3. Уроки которые почти никто не смотрел
|
||||
SELECT
|
||||
lesson_title,
|
||||
course_title,
|
||||
total_views,
|
||||
unique_users
|
||||
FROM lesson_popularity_summary
|
||||
WHERE total_views <= 1
|
||||
ORDER BY total_views ASC;
|
||||
|
||||
-- Неактивные пользователи без записей на курсы
|
||||
SELECT
|
||||
user_name,
|
||||
email,
|
||||
age,
|
||||
registration_date
|
||||
FROM inactive_users_summary
|
||||
WHERE registered_courses_count = 0
|
||||
ORDER BY registration_date DESC;
|
||||
|
||||
-- Пользователи, полностью завершившие курс
|
||||
SELECT
|
||||
user_name,
|
||||
course_title,
|
||||
lessons_in_course,
|
||||
lessons_viewed,
|
||||
completion_rate
|
||||
FROM course_completion_rate
|
||||
WHERE completion_rate = 1
|
||||
ORDER BY course_title, user_name;
|
||||
|
||||
/* Отчет 1: При сравнении витрины lesson_popularity_summary в ClickHouse и Greenplum результаты совпали только частично
|
||||
Совпали показатели:
|
||||
SUM(total_views) = 58
|
||||
MAX(total_views) = 9
|
||||
|
||||
Не совпали показатели:
|
||||
AVG(total_views)
|
||||
MIN(total_views)
|
||||
|
||||
В Clickhouse:
|
||||
AVG(total_views) = 3.2222
|
||||
MIN(total_views) = 0
|
||||
В Greenplum:
|
||||
AVG(total_views) = 3.8666
|
||||
MIN(total_views) = 1
|
||||
|
||||
Причина расхождения: в ClickHouse в витрине есть дополнительные строки с total_views = 0, а в Greenplum таких строк нет. Из-за этих нулевых строк минимальное значение в ClickHouse стало 0, а среднее значение уменьшилось
|
||||
|
||||
|
||||
Отчет 2: При проверке витрины inactive_users_summary результаты в ClickHouse и Greenplum совпали полностью.
|
||||
В обеих системах получены одинаковые значения:
|
||||
inactive_users_count = 9
|
||||
sum_registered_courses = 17
|
||||
avg_registered_courses = 1.8888888889
|
||||
min_registered_courses = 1
|
||||
max_registered_courses = 3
|
||||
|
||||
*/
|
||||
Reference in New Issue
Block a user