Batirbulat
410 lines
12 KiB
SQL
410 lines
12 KiB
SQL
--ПЕТ ПРОЕКТ СОЗДАНИЕ ТАБЛИЦ
|
|
|
|
CREATE TABLE bhabibullin.core_enrollments
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
user_id UInt32,
|
|
course_id UInt32,
|
|
enrolled_at DateTime
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/core_enrollments',
|
|
'{replica}'
|
|
)
|
|
ORDER BY (user_id, enrolled_at);
|
|
|
|
|
|
CREATE TABLE bhabibullin.core_dim_user
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
user_id UInt32,
|
|
name String,
|
|
age UInt32,
|
|
email String,
|
|
registration_date Date
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/core_dim_user',
|
|
'{replica}'
|
|
)
|
|
ORDER BY (user_id, registration_date);
|
|
|
|
|
|
CREATE TABLE bhabibullin.core_fact_lesson_views
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
user_id UInt32,
|
|
lesson_id UInt32,
|
|
course_id UInt32,
|
|
viewed_at DateTime64(3)
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/core_fact_lesson_views',
|
|
'{replica}'
|
|
)
|
|
ORDER BY user_id;
|
|
|
|
CREATE TABLE bhabibullin.core_dim_course
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
course_id UInt32,
|
|
title String,
|
|
category String
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/core_dim_course',
|
|
'{replica}'
|
|
)
|
|
ORDER BY course_id;
|
|
|
|
drop table bhabibullin.core_dim_course ;
|
|
|
|
CREATE TABLE bhabibullin.core_dim_lesson
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
lesson_id UInt32,
|
|
title String,
|
|
duration_min UInt32,
|
|
course_id UInt32
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/core_dim_lesson',
|
|
'{replica}'
|
|
)
|
|
ORDER BY (lesson_id, course_id);
|
|
|
|
|
|
-- Этап 2. Создание витринных таблиц
|
|
-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
|
|
/*
|
|
- Спроектируй таблицу, где каждая строка — один урок.
|
|
- Для каждого урока должны храниться:
|
|
- ID и название урока
|
|
- ID и название курса, к которому он относится
|
|
- Общее число просмотров
|
|
- Количество уникальных пользователей, посмотревших урок
|
|
- Дата первого и последнего просмотра
|
|
*/
|
|
|
|
CREATE TABLE bhabibullin.lesson_popularity_summary
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
lesson_id UInt32,
|
|
lesson_title String,
|
|
course_id UInt32,
|
|
course_title String,
|
|
total_views UInt32,
|
|
unique_users UInt32,
|
|
first_view DateTime64(3, 'UTC'),
|
|
last_view DateTime64(3, 'UTC')
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary',
|
|
'{replica}'
|
|
)
|
|
ORDER BY (lesson_id, course_id);
|
|
|
|
CREATE MATERIALIZED VIEW mv_lesson_popularity_summary
|
|
ON CLUSTER clickhouse_cluster
|
|
TO bhabibullin.lesson_popularity_summary
|
|
AS
|
|
SELECT
|
|
flv.lesson_id AS lesson_id,
|
|
l.title AS lesson_title,
|
|
flv.course_id AS course_id,
|
|
c.title AS course_title,
|
|
count() AS total_views,
|
|
uniqExact(flv.user_id) AS unique_users,
|
|
min(flv.viewed_at) AS first_view,
|
|
max(flv.viewed_at) AS last_view
|
|
FROM core_fact_lesson_views AS flv
|
|
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
|
|
JOIN core_dim_course AS c ON flv.course_id = c.course_id
|
|
GROUP BY
|
|
flv.lesson_id,
|
|
l.title,
|
|
flv.course_id,
|
|
c.title;
|
|
|
|
INSERT INTO bhabibullin.lesson_popularity_summary
|
|
SELECT
|
|
flv.lesson_id,
|
|
l.title AS lesson_title,
|
|
flv.course_id,
|
|
c.title AS course_title,
|
|
count() AS total_views,
|
|
uniqExact(flv.user_id) AS unique_users,
|
|
min(flv.viewed_at) AS first_view,
|
|
max(flv.viewed_at) AS last_view
|
|
FROM core_fact_lesson_views AS flv
|
|
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
|
|
JOIN core_dim_course AS c ON flv.course_id = c.course_id
|
|
GROUP BY
|
|
flv.lesson_id,
|
|
l.title,
|
|
flv.course_id,
|
|
c.title;
|
|
|
|
select * from bhabibullin.mv_lesson_popularity_summary ;
|
|
select * from bhabibullin.lesson_popularity_summary ;
|
|
show tables;
|
|
|
|
/*
|
|
### ✅ **Задание 3: Создай витрину `inactive_users_summary`**
|
|
|
|
**Цель:** выявить пользователей, которые **зарегистрировались, но ничего не посмотрели**.
|
|
|
|
📌 **Что нужно сделать:**
|
|
|
|
- Подготовь таблицу, где каждая строка — один такой "неактивный" пользователь.
|
|
- Для каждого пользователя нужны:
|
|
- ID, имя, email, возраст
|
|
- Количество курсов, на которые он записался
|
|
- Дата регистрации
|
|
|
|
**Где пригодится:**
|
|
|
|
Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости.
|
|
*/
|
|
|
|
DROP TABLE bhabibullin.inactive_users_summary;
|
|
DROP VIEW bhabibullin.mv_inactive_users_summary ;
|
|
show tables;
|
|
|
|
CREATE TABLE bhabibullin.inactive_users_summary
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
user_id UInt32,
|
|
name String,
|
|
email String,
|
|
age UInt32,
|
|
enrollment_count UInt32,
|
|
registration_date Date
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/inactive_users_summary',
|
|
'{replica}'
|
|
)
|
|
ORDER BY user_id;
|
|
|
|
CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary
|
|
ON CLUSTER clickhouse_cluster
|
|
to bhabibullin.inactive_users_summary
|
|
AS
|
|
SELECT
|
|
cdu.user_id,
|
|
cdu.name,
|
|
cdu.email,
|
|
cdu.age,
|
|
0 as enrollment_count,
|
|
cdu.registration_date
|
|
FROM core_dim_user cdu
|
|
WHERE (cdu.user_id, 1) NOT IN (
|
|
SELECT user_id, 1
|
|
FROM core_enrollments
|
|
GROUP BY user_id
|
|
);
|
|
|
|
INSERT INTO bhabibullin.inactive_users_summary
|
|
SELECT
|
|
cdu.user_id,
|
|
cdu.name,
|
|
cdu.email,
|
|
cdu.age,
|
|
0 as enrollment_count,
|
|
cdu.registration_date
|
|
FROM core_dim_user cdu
|
|
WHERE (cdu.user_id, 1) NOT IN (
|
|
SELECT user_id, 1
|
|
FROM core_enrollments
|
|
GROUP BY user_id
|
|
);
|
|
|
|
select * from bhabibullin.inactive_users_summary ius ;
|
|
select * from bhabibullin.mv_inactive_users_summary mius ;
|
|
|
|
/*
|
|
### ✅ **Задание 4: Создай витрину `course_completion_rate`**
|
|
|
|
**Цель:** проанализировать, насколько хорошо пользователи проходят курсы.
|
|
|
|
📌 **Что нужно сделать:**
|
|
|
|
- Сформируй таблицу, где строка — это пользователь + курс.
|
|
- Для каждой пары нужно рассчитать:
|
|
- Общее число уроков в курсе
|
|
- Сколько из них просмотрел конкретный пользователь
|
|
- Коэффициент завершения (`lessons_viewed / lessons_in_course`)
|
|
*/
|
|
|
|
CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate
|
|
ON CLUSTER clickhouse_cluster
|
|
(
|
|
user_id UInt32,
|
|
user_name String,
|
|
course_id UInt32,
|
|
course_title String,
|
|
lessons_in_course UInt32,
|
|
lessons_viewed UInt32,
|
|
completion_rate Float64
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/clickhouse/tables/{shard}/bhabibullin/course_completion_rate',
|
|
'{replica}'
|
|
)
|
|
ORDER BY (user_id, course_id);
|
|
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate
|
|
ON CLUSTER clickhouse_cluster
|
|
TO bhabibullin.course_completion_rate
|
|
AS
|
|
WITH lessons_in_course AS (
|
|
SELECT
|
|
course_id,
|
|
count(Distinct lesson_id) AS lessons_in_course
|
|
FROM core_dim_lesson
|
|
GROUP BY course_id
|
|
),
|
|
views_user_course AS (
|
|
SELECT
|
|
user_id,
|
|
course_id,
|
|
count(Distinct lesson_id) AS lessons_viewed
|
|
FROM core_fact_lesson_views
|
|
GROUP BY user_id, course_id
|
|
)
|
|
SELECT
|
|
u.user_id AS user_id,
|
|
u.name AS user_name,
|
|
c.course_id AS course_id,
|
|
c.title AS course_title,
|
|
lic.lessons_in_course AS lessons_in_course,
|
|
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
|
|
ROUND(
|
|
COALESCE(
|
|
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
|
|
/ NULLIF(lic.lessons_in_course * 1.0, 0),
|
|
0
|
|
),
|
|
2
|
|
) AS completion_rate
|
|
FROM core_dim_user u
|
|
JOIN core_enrollments e ON u.user_id = e.user_id
|
|
JOIN core_dim_course c ON e.course_id = c.course_id
|
|
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
|
|
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id
|
|
ORDER BY
|
|
u.user_id,
|
|
c.course_id;
|
|
|
|
INSERT INTO bhabibullin.course_completion_rate
|
|
WITH lessons_in_course AS (
|
|
SELECT
|
|
course_id,
|
|
count(Distinct lesson_id) AS lessons_in_course
|
|
FROM core_dim_lesson
|
|
GROUP BY course_id
|
|
),
|
|
views_user_course AS (
|
|
SELECT
|
|
user_id,
|
|
course_id,
|
|
count(Distinct lesson_id) AS lessons_viewed
|
|
FROM core_fact_lesson_views
|
|
GROUP BY user_id, course_id
|
|
)
|
|
SELECT
|
|
u.user_id AS user_id,
|
|
u.name AS user_name,
|
|
c.course_id AS course_id,
|
|
c.title AS course_title,
|
|
lic.lessons_in_course AS lessons_in_course,
|
|
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
|
|
ROUND(
|
|
COALESCE(
|
|
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
|
|
/ NULLIF(lic.lessons_in_course * 1.0, 0),
|
|
0
|
|
),
|
|
2
|
|
) AS completion_rate
|
|
FROM core_dim_user u
|
|
JOIN core_enrollments e ON u.user_id = e.user_id
|
|
JOIN core_dim_course c ON e.course_id = c.course_id
|
|
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
|
|
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id;
|
|
|
|
select * from bhabibullin.course_completion_rate;
|
|
select * from bhabibullin.mv_course_completion_rate;
|
|
|
|
/*
|
|
# 📊 Этап 3. Анализ и запросы
|
|
|
|
### ✅ **Задание 5: Напиши SQL-запросы к витринам**
|
|
|
|
**Напиши свои вариации для практики.**
|
|
*/
|
|
|
|
--Топ-5 самых просматриваемых уроков
|
|
SELECT
|
|
lesson_title,
|
|
course_title,
|
|
total_views
|
|
FROM lesson_popularity_summary
|
|
ORDER BY total_views DESC
|
|
LIMIT 5;
|
|
|
|
--Неактивные пользователи, записавшиеся на курсы
|
|
SELECT
|
|
name,
|
|
email,
|
|
enrollment_count
|
|
FROM inactive_users_summary
|
|
WHERE enrollment_count > 0
|
|
ORDER BY registration_date DESC;
|
|
|
|
--Курсы с самым высоким процентом завершения
|
|
SELECT
|
|
course_title,
|
|
AVG(completion_rate) AS avg_completion
|
|
FROM course_completion_rate
|
|
GROUP BY course_title
|
|
ORDER BY avg_completion DESC
|
|
LIMIT 5;
|
|
|
|
-- топ 5 курсов по количеству уникальных пользователей
|
|
SELECT
|
|
course_title,
|
|
unique_users
|
|
FROM lesson_popularity_summary
|
|
ORDER BY unique_users DESC
|
|
LIMIT 5;
|
|
|
|
--новые пользователи за прошлый месяц
|
|
SELECT
|
|
user_id,
|
|
name,
|
|
email,
|
|
registration_date
|
|
FROM core_dim_user
|
|
WHERE registration_date >= today() - INTERVAL 30 DAY
|
|
ORDER BY registration_date DESC;
|
|
|
|
/*
|
|
### ✅ **Задание 6: Сравни данные с Greenplum**
|
|
|
|
1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
|
|
2. Сравни с ClickHouse-результатами:
|
|
- Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
|
|
- Проверяй `NULL`, `0`, округления
|
|
3. Напиши мини-отчёт: **"где совпадает, где нет, почему"
|
|
|
|
Совпадает ли SUM, AVG, MIN, MAX
|
|
Совпадает и в гринплам, и в кликхаус
|
|
|
|
Проверяй NULL, 0, округления
|
|
С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения
|
|
*/
|
|
|