Batirbulat
416 lines
12 KiB
SQL
416 lines
12 KiB
SQL
--ПЕТ ПРОЕКТ СОЗДАНИЕ ТАБЛИЦ
|
||
|
||
CREATE TABLE bhabibullin.core_enrollments
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id UInt32,
|
||
course_id UInt32,
|
||
enrolled_at DateTime
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/core_enrollments',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (user_id, enrolled_at);
|
||
|
||
|
||
CREATE TABLE bhabibullin.core_dim_user
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id UInt32,
|
||
name String,
|
||
age UInt32,
|
||
email String,
|
||
registration_date Date
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/core_dim_user',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (user_id, registration_date);
|
||
|
||
|
||
CREATE TABLE bhabibullin.core_fact_lesson_views
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id UInt32,
|
||
lesson_id UInt32,
|
||
course_id UInt32,
|
||
viewed_at DateTime64(3)
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/core_fact_lesson_views',
|
||
'{replica}'
|
||
)
|
||
ORDER BY user_id;
|
||
|
||
CREATE TABLE bhabibullin.core_dim_course
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
course_id UInt32,
|
||
title String,
|
||
category String
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/core_dim_course',
|
||
'{replica}'
|
||
)
|
||
ORDER BY course_id;
|
||
|
||
drop table bhabibullin.core_dim_course ;
|
||
|
||
CREATE TABLE bhabibullin.core_dim_lesson
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
lesson_id UInt32,
|
||
title String,
|
||
duration_min UInt32,
|
||
course_id UInt32
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/core_dim_lesson',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (lesson_id, course_id);
|
||
|
||
|
||
-- Этап 2. Создание витринных таблиц
|
||
-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
|
||
/*
|
||
- Спроектируй таблицу, где каждая строка — один урок.
|
||
- Для каждого урока должны храниться:
|
||
- ID и название урока
|
||
- ID и название курса, к которому он относится
|
||
- Общее число просмотров
|
||
- Количество уникальных пользователей, посмотревших урок
|
||
- Дата первого и последнего просмотра
|
||
*/
|
||
|
||
-- Основная таблица с AggregatingMergeTree
|
||
|
||
CREATE TABLE bhabibullin.lesson_popularity_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
lesson_id UInt32,
|
||
lesson_title String,
|
||
course_id UInt32,
|
||
course_title String,
|
||
total_views AggregateFunction(sum, UInt64),
|
||
unique_users AggregateFunction(uniq, UInt32),
|
||
first_view AggregateFunction(min, DateTime64(3, 'UTC')),
|
||
last_view AggregateFunction(max, DateTime64(3, 'UTC'))
|
||
)
|
||
ENGINE = ReplicatedAggregatingMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (lesson_id, course_id)
|
||
SETTINGS index_granularity = 8192;
|
||
|
||
CREATE MATERIALIZED VIEW bhabibullin.mv_lesson_popularity_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
TO bhabibullin.lesson_popularity_summary
|
||
AS
|
||
SELECT
|
||
flv.lesson_id AS lesson_id,
|
||
l.title AS lesson_title,
|
||
flv.course_id AS course_id,
|
||
c.title AS course_title,
|
||
sumState(1) AS total_views,
|
||
uniqState(flv.user_id) AS unique_users,
|
||
minState(flv.viewed_at) AS first_view,
|
||
maxState(flv.viewed_at) AS last_view
|
||
FROM core_fact_lesson_views AS flv
|
||
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
|
||
JOIN core_dim_course AS c ON flv.course_id = c.course_id
|
||
GROUP BY
|
||
flv.lesson_id,
|
||
l.title,
|
||
flv.course_id,
|
||
c.title;
|
||
|
||
INSERT INTO bhabibullin.lesson_popularity_summary
|
||
SELECT
|
||
flv.lesson_id,
|
||
l.title AS lesson_title,
|
||
flv.course_id,
|
||
c.title AS course_title,
|
||
count() AS total_views,
|
||
uniqExact(flv.user_id) AS unique_users,
|
||
min(flv.viewed_at) AS first_view,
|
||
max(flv.viewed_at) AS last_view
|
||
FROM core_fact_lesson_views AS flv
|
||
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
|
||
JOIN core_dim_course AS c ON flv.course_id = c.course_id
|
||
GROUP BY
|
||
flv.lesson_id,
|
||
l.title,
|
||
flv.course_id,
|
||
c.title;
|
||
|
||
select * from bhabibullin.mv_lesson_popularity_summary ;
|
||
select * from bhabibullin.lesson_popularity_summary ;
|
||
show tables;
|
||
|
||
/*
|
||
### ✅ **Задание 3: Создай витрину `inactive_users_summary`**
|
||
|
||
**Цель:** выявить пользователей, которые **зарегистрировались, но ничего не посмотрели**.
|
||
|
||
📌 **Что нужно сделать:**
|
||
|
||
- Подготовь таблицу, где каждая строка — один такой "неактивный" пользователь.
|
||
- Для каждого пользователя нужны:
|
||
- ID, имя, email, возраст
|
||
- Количество курсов, на которые он записался
|
||
- Дата регистрации
|
||
|
||
**Где пригодится:**
|
||
|
||
Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости.
|
||
*/
|
||
|
||
DROP TABLE bhabibullin.inactive_users_summary;
|
||
DROP VIEW bhabibullin.mv_inactive_users_summary ;
|
||
show tables;
|
||
|
||
CREATE TABLE bhabibullin.inactive_users_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id UInt32,
|
||
name String,
|
||
email String,
|
||
age UInt32,
|
||
enrollment_count UInt32,
|
||
registration_date Date
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/inactive_users_summary',
|
||
'{replica}'
|
||
)
|
||
ORDER BY user_id;
|
||
|
||
CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
TO bhabibullin.inactive_users_summary
|
||
AS
|
||
SELECT
|
||
cdu.user_id,
|
||
cdu.name,
|
||
cdu.email,
|
||
cdu.age,
|
||
countDistinct(flv.course_id) as enrollment_count,
|
||
cdu.registration_date
|
||
FROM core_dim_user cdu
|
||
LEFT ANTI JOIN core_fact_lesson_views flv
|
||
ON cdu.user_id = flv.user_id
|
||
GROUP BY
|
||
cdu.user_id,
|
||
cdu.name,
|
||
cdu.email,
|
||
cdu.age,
|
||
cdu.registration_date;
|
||
|
||
INSERT INTO bhabibullin.inactive_users_summary
|
||
SELECT
|
||
cdu.user_id,
|
||
cdu.name,
|
||
cdu.email,
|
||
cdu.age,
|
||
0 as enrollment_count,
|
||
cdu.registration_date
|
||
FROM core_dim_user cdu
|
||
WHERE (cdu.user_id, 1) NOT IN (
|
||
SELECT user_id, 1
|
||
FROM core_enrollments
|
||
GROUP BY user_id
|
||
);
|
||
|
||
select * from bhabibullin.inactive_users_summary ius ;
|
||
select * from bhabibullin.mv_inactive_users_summary mius ;
|
||
|
||
/*
|
||
### ✅ **Задание 4: Создай витрину `course_completion_rate`**
|
||
|
||
**Цель:** проанализировать, насколько хорошо пользователи проходят курсы.
|
||
|
||
📌 **Что нужно сделать:**
|
||
|
||
- Сформируй таблицу, где строка — это пользователь + курс.
|
||
- Для каждой пары нужно рассчитать:
|
||
- Общее число уроков в курсе
|
||
- Сколько из них просмотрел конкретный пользователь
|
||
- Коэффициент завершения (`lessons_viewed / lessons_in_course`)
|
||
*/
|
||
|
||
CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id UInt32,
|
||
user_name String,
|
||
course_id UInt32,
|
||
course_title String,
|
||
lessons_in_course UInt32,
|
||
lessons_viewed UInt32,
|
||
completion_rate Float64
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/bhabibullin/course_completion_rate',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (user_id, course_id);
|
||
|
||
|
||
CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate
|
||
ON CLUSTER clickhouse_cluster
|
||
TO bhabibullin.course_completion_rate
|
||
AS
|
||
WITH lessons_in_course AS (
|
||
SELECT
|
||
course_id,
|
||
count(Distinct lesson_id) AS lessons_in_course
|
||
FROM core_dim_lesson
|
||
GROUP BY course_id
|
||
),
|
||
views_user_course AS (
|
||
SELECT
|
||
user_id,
|
||
course_id,
|
||
count(Distinct lesson_id) AS lessons_viewed
|
||
FROM core_fact_lesson_views
|
||
GROUP BY user_id, course_id
|
||
)
|
||
SELECT
|
||
u.user_id AS user_id,
|
||
u.name AS user_name,
|
||
c.course_id AS course_id,
|
||
c.title AS course_title,
|
||
lic.lessons_in_course AS lessons_in_course,
|
||
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
|
||
ROUND(
|
||
COALESCE(
|
||
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
|
||
/ NULLIF(lic.lessons_in_course * 1.0, 0),
|
||
0
|
||
),
|
||
2
|
||
) AS completion_rate
|
||
FROM core_dim_user u
|
||
JOIN core_enrollments e ON u.user_id = e.user_id
|
||
JOIN core_dim_course c ON e.course_id = c.course_id
|
||
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
|
||
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id
|
||
ORDER BY
|
||
u.user_id,
|
||
c.course_id;
|
||
|
||
INSERT INTO bhabibullin.course_completion_rate
|
||
WITH lessons_in_course AS (
|
||
SELECT
|
||
course_id,
|
||
count(Distinct lesson_id) AS lessons_in_course
|
||
FROM core_dim_lesson
|
||
GROUP BY course_id
|
||
),
|
||
views_user_course AS (
|
||
SELECT
|
||
user_id,
|
||
course_id,
|
||
count(Distinct lesson_id) AS lessons_viewed
|
||
FROM core_fact_lesson_views
|
||
GROUP BY user_id, course_id
|
||
)
|
||
SELECT
|
||
u.user_id AS user_id,
|
||
u.name AS user_name,
|
||
c.course_id AS course_id,
|
||
c.title AS course_title,
|
||
lic.lessons_in_course AS lessons_in_course,
|
||
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
|
||
ROUND(
|
||
COALESCE(
|
||
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
|
||
/ NULLIF(lic.lessons_in_course * 1.0, 0),
|
||
0
|
||
),
|
||
2
|
||
) AS completion_rate
|
||
FROM core_dim_user u
|
||
JOIN core_enrollments e ON u.user_id = e.user_id
|
||
JOIN core_dim_course c ON e.course_id = c.course_id
|
||
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
|
||
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id;
|
||
|
||
select * from bhabibullin.course_completion_rate;
|
||
select * from bhabibullin.mv_course_completion_rate;
|
||
|
||
/*
|
||
# 📊 Этап 3. Анализ и запросы
|
||
|
||
### ✅ **Задание 5: Напиши SQL-запросы к витринам**
|
||
|
||
**Напиши свои вариации для практики.**
|
||
*/
|
||
|
||
--Топ-5 самых просматриваемых уроков
|
||
SELECT
|
||
lesson_title,
|
||
course_title,
|
||
total_views
|
||
FROM lesson_popularity_summary
|
||
ORDER BY total_views DESC
|
||
LIMIT 5;
|
||
|
||
--Неактивные пользователи, записавшиеся на курсы
|
||
SELECT
|
||
name,
|
||
email,
|
||
enrollment_count
|
||
FROM inactive_users_summary
|
||
WHERE enrollment_count > 0
|
||
ORDER BY registration_date DESC;
|
||
|
||
--Курсы с самым высоким процентом завершения
|
||
SELECT
|
||
course_title,
|
||
AVG(completion_rate) AS avg_completion
|
||
FROM course_completion_rate
|
||
GROUP BY course_title
|
||
ORDER BY avg_completion DESC
|
||
LIMIT 5;
|
||
|
||
-- топ 5 курсов по количеству уникальных пользователей
|
||
SELECT
|
||
course_title,
|
||
unique_users
|
||
FROM lesson_popularity_summary
|
||
ORDER BY unique_users DESC
|
||
LIMIT 5;
|
||
|
||
--новые пользователи за прошлый месяц
|
||
SELECT
|
||
user_id,
|
||
name,
|
||
email,
|
||
registration_date
|
||
FROM core_dim_user
|
||
WHERE registration_date >= today() - INTERVAL 30 DAY
|
||
ORDER BY registration_date DESC;
|
||
|
||
/*
|
||
### ✅ **Задание 6: Сравни данные с Greenplum**
|
||
|
||
1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
|
||
2. Сравни с ClickHouse-результатами:
|
||
- Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
|
||
- Проверяй `NULL`, `0`, округления
|
||
3. Напиши мини-отчёт: **"где совпадает, где нет, почему"
|
||
|
||
Совпадает ли SUM, AVG, MIN, MAX
|
||
Совпадает и в гринплам, и в кликхаус
|
||
|
||
Проверяй NULL, 0, округления
|
||
С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения
|
||
*/
|
||
|