Не менял ничего
This commit is contained in:
Batirbulat
@@ -0,0 +1,384 @@
|
||||
--ПЕТ ПРОЕКТ СОЗДАНИЕ ТАБЛИЦ
|
||||
|
||||
CREATE TABLE bhabibullin.core_enrollments
|
||||
ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id UInt32,
|
||||
course_id UInt32,
|
||||
enrolled_at DateTime
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/bhabibullin/core_enrollments',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (user_id, enrolled_at);
|
||||
|
||||
|
||||
CREATE TABLE bhabibullin.core_dim_user
|
||||
ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id UInt32,
|
||||
name String,
|
||||
age UInt32,
|
||||
email String,
|
||||
registration_date Date
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/bhabibullin/core_dim_user',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (user_id, registration_date);
|
||||
|
||||
|
||||
CREATE TABLE bhabibullin.core_fact_lesson_views
|
||||
ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
user_id UInt32,
|
||||
lesson_id UInt32,
|
||||
course_id UInt32,
|
||||
viewed_at DateTime64(3)
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/bhabibullin/core_fact_lesson_views',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY user_id;
|
||||
|
||||
CREATE TABLE bhabibullin.core_dim_course
|
||||
ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
course_id UInt32,
|
||||
title String,
|
||||
category String
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/bhabibullin/core_dim_course',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY course_id;
|
||||
|
||||
drop table bhabibullin.core_dim_course ;
|
||||
|
||||
CREATE TABLE bhabibullin.core_dim_lesson
|
||||
ON CLUSTER clickhouse_cluster
|
||||
(
|
||||
lesson_id UInt32,
|
||||
title String,
|
||||
duration_min UInt32,
|
||||
course_id UInt32
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/bhabibullin/core_dim_lesson',
|
||||
'{replica}'
|
||||
)
|
||||
ORDER BY (lesson_id, course_id);
|
||||
|
||||
|
||||
-- Этап 2. Создание витринных таблиц
|
||||
-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
|
||||
/*
|
||||
- Спроектируй таблицу, где каждая строка — один урок.
|
||||
- Для каждого урока должны храниться:
|
||||
- ID и название урока
|
||||
- ID и название курса, к которому он относится
|
||||
- Общее число просмотров
|
||||
- Количество уникальных пользователей, посмотревших урок
|
||||
- Дата первого и последнего просмотра
|
||||
*/
|
||||
|
||||
CREATE TABLE bhabibullin.lesson_popularity_summary
|
||||
(
|
||||
lesson_id UInt32,
|
||||
lesson_title String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
total_views UInt32,
|
||||
unique_users UInt32,
|
||||
first_view DateTime64(3, 'UTC'),
|
||||
last_view DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (lesson_id, course_id);
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_lesson_popularity_summary
|
||||
TO bhabibullin.lesson_popularity_summary
|
||||
AS
|
||||
SELECT
|
||||
flv.lesson_id AS lesson_id,
|
||||
l.title AS lesson_title,
|
||||
flv.course_id AS course_id,
|
||||
c.title AS course_title,
|
||||
count() AS total_views,
|
||||
uniqExact(flv.user_id) AS unique_users,
|
||||
min(flv.viewed_at) AS first_view,
|
||||
max(flv.viewed_at) AS last_view
|
||||
FROM core_fact_lesson_views AS flv
|
||||
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
|
||||
JOIN core_dim_course AS c ON flv.course_id = c.course_id
|
||||
GROUP BY
|
||||
flv.lesson_id,
|
||||
l.title,
|
||||
flv.course_id,
|
||||
c.title;
|
||||
|
||||
INSERT INTO bhabibullin.lesson_popularity_summary
|
||||
SELECT
|
||||
flv.lesson_id,
|
||||
l.title AS lesson_title,
|
||||
flv.course_id,
|
||||
c.title AS course_title,
|
||||
count() AS total_views,
|
||||
uniqExact(flv.user_id) AS unique_users,
|
||||
min(flv.viewed_at) AS first_view,
|
||||
max(flv.viewed_at) AS last_view
|
||||
FROM core_fact_lesson_views AS flv
|
||||
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
|
||||
JOIN core_dim_course AS c ON flv.course_id = c.course_id
|
||||
GROUP BY
|
||||
flv.lesson_id,
|
||||
l.title,
|
||||
flv.course_id,
|
||||
c.title;
|
||||
|
||||
select * from bhabibullin.mv_lesson_popularity_summary mlps ;
|
||||
select * from bhabibullin.lesson_popularity_summary lps ;
|
||||
|
||||
/*
|
||||
### ✅ **Задание 3: Создай витрину `inactive_users_summary`**
|
||||
|
||||
**Цель:** выявить пользователей, которые **зарегистрировались, но ничего не посмотрели**.
|
||||
|
||||
📌 **Что нужно сделать:**
|
||||
|
||||
- Подготовь таблицу, где каждая строка — один такой "неактивный" пользователь.
|
||||
- Для каждого пользователя нужны:
|
||||
- ID, имя, email, возраст
|
||||
- Количество курсов, на которые он записался
|
||||
- Дата регистрации
|
||||
|
||||
**Где пригодится:**
|
||||
|
||||
Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости.
|
||||
*/
|
||||
|
||||
CREATE TABLE bhabibullin.inactive_users_summary(
|
||||
user_id UInt32,
|
||||
name String,
|
||||
email String,
|
||||
age UInt32,
|
||||
enrollment_count UInt32,
|
||||
registration_date Date
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY user_id;
|
||||
|
||||
CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary
|
||||
to bhabibullin.inactive_users_summary
|
||||
AS
|
||||
SELECT
|
||||
cdu.user_id,
|
||||
cdu.name,
|
||||
cdu.email,
|
||||
cdu.age,
|
||||
0 as enrollment_count,
|
||||
cdu.registration_date
|
||||
FROM core_dim_user cdu
|
||||
WHERE (cdu.user_id, 1) NOT IN (
|
||||
SELECT user_id, 1
|
||||
FROM core_enrollments
|
||||
GROUP BY user_id
|
||||
);
|
||||
|
||||
INSERT INTO bhabibullin.inactive_users_summary
|
||||
SELECT
|
||||
cdu.user_id,
|
||||
cdu.name,
|
||||
cdu.email,
|
||||
cdu.age,
|
||||
0 as enrollment_count,
|
||||
cdu.registration_date
|
||||
FROM core_dim_user cdu
|
||||
WHERE (cdu.user_id, 1) NOT IN (
|
||||
SELECT user_id, 1
|
||||
FROM core_enrollments
|
||||
GROUP BY user_id
|
||||
);
|
||||
|
||||
/*
|
||||
### ✅ **Задание 4: Создай витрину `course_completion_rate`**
|
||||
|
||||
**Цель:** проанализировать, насколько хорошо пользователи проходят курсы.
|
||||
|
||||
📌 **Что нужно сделать:**
|
||||
|
||||
- Сформируй таблицу, где строка — это пользователь + курс.
|
||||
- Для каждой пары нужно рассчитать:
|
||||
- Общее число уроков в курсе
|
||||
- Сколько из них просмотрел конкретный пользователь
|
||||
- Коэффициент завершения (`lessons_viewed / lessons_in_course`)
|
||||
*/
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate
|
||||
(
|
||||
user_id UInt32,
|
||||
user_name String,
|
||||
course_id UInt32,
|
||||
course_title String,
|
||||
lessons_in_course UInt32,
|
||||
lessons_viewed UInt32,
|
||||
completion_rate Float64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (user_id, course_id);
|
||||
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate
|
||||
TO bhabibullin.course_completion_rate
|
||||
AS
|
||||
WITH lessons_in_course AS (
|
||||
SELECT
|
||||
course_id,
|
||||
count(Distinct lesson_id) AS lessons_in_course
|
||||
FROM core_dim_lesson
|
||||
GROUP BY course_id
|
||||
),
|
||||
views_user_course AS (
|
||||
SELECT
|
||||
user_id,
|
||||
course_id,
|
||||
count(Distinct lesson_id) AS lessons_viewed
|
||||
FROM core_fact_lesson_views
|
||||
GROUP BY user_id, course_id
|
||||
)
|
||||
SELECT
|
||||
u.user_id AS user_id,
|
||||
u.name AS user_name,
|
||||
c.course_id AS course_id,
|
||||
c.title AS course_title,
|
||||
lic.lessons_in_course AS lessons_in_course,
|
||||
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
|
||||
ROUND(
|
||||
COALESCE(
|
||||
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
|
||||
/ NULLIF(lic.lessons_in_course * 1.0, 0),
|
||||
0
|
||||
),
|
||||
2
|
||||
) AS completion_rate
|
||||
FROM core_dim_user u
|
||||
JOIN core_enrollments e ON u.user_id = e.user_id
|
||||
JOIN core_dim_course c ON e.course_id = c.course_id
|
||||
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
|
||||
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id
|
||||
ORDER BY
|
||||
u.user_id,
|
||||
c.course_id;
|
||||
|
||||
INSERT INTO bhabibullin.course_completion_rate
|
||||
WITH lessons_in_course AS (
|
||||
SELECT
|
||||
course_id,
|
||||
count(Distinct lesson_id) AS lessons_in_course
|
||||
FROM core_dim_lesson
|
||||
GROUP BY course_id
|
||||
),
|
||||
views_user_course AS (
|
||||
SELECT
|
||||
user_id,
|
||||
course_id,
|
||||
count(Distinct lesson_id) AS lessons_viewed
|
||||
FROM core_fact_lesson_views
|
||||
GROUP BY user_id, course_id
|
||||
)
|
||||
SELECT
|
||||
u.user_id AS user_id,
|
||||
u.name AS user_name,
|
||||
c.course_id AS course_id,
|
||||
c.title AS course_title,
|
||||
lic.lessons_in_course AS lessons_in_course,
|
||||
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
|
||||
ROUND(
|
||||
COALESCE(
|
||||
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
|
||||
/ NULLIF(lic.lessons_in_course * 1.0, 0),
|
||||
0
|
||||
),
|
||||
2
|
||||
) AS completion_rate
|
||||
FROM core_dim_user u
|
||||
JOIN core_enrollments e ON u.user_id = e.user_id
|
||||
JOIN core_dim_course c ON e.course_id = c.course_id
|
||||
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
|
||||
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id;
|
||||
|
||||
/*
|
||||
# 📊 Этап 3. Анализ и запросы
|
||||
|
||||
### ✅ **Задание 5: Напиши SQL-запросы к витринам**
|
||||
|
||||
**Напиши свои вариации для практики.**
|
||||
*/
|
||||
|
||||
--Топ-5 самых просматриваемых уроков
|
||||
SELECT
|
||||
lesson_title,
|
||||
course_title,
|
||||
total_views
|
||||
FROM lesson_popularity_summary
|
||||
ORDER BY total_views DESC
|
||||
LIMIT 5;
|
||||
|
||||
--Неактивные пользователи, записавшиеся на курсы
|
||||
SELECT
|
||||
name,
|
||||
email,
|
||||
registered_courses_count
|
||||
FROM inactive_users_summary
|
||||
WHERE registered_courses_count > 0
|
||||
ORDER BY registration_date DESC;
|
||||
|
||||
--Курсы с самым высоким процентом завершения
|
||||
SELECT
|
||||
course_title,
|
||||
AVG(completion_rate) AS avg_completion
|
||||
FROM course_completion_rate
|
||||
GROUP BY course_title
|
||||
ORDER BY avg_completion DESC
|
||||
LIMIT 5;
|
||||
|
||||
-- топ 5 курсов по количеству уникальных пользователей
|
||||
SELECT
|
||||
course_title,
|
||||
uniqExact(user_id) AS active_users
|
||||
FROM lesson_popularity_summary
|
||||
GROUP BY course_title
|
||||
ORDER BY active_users DESC
|
||||
LIMIT 5;
|
||||
|
||||
--новые пользователи за прошлый месяц
|
||||
SELECT
|
||||
user_id,
|
||||
name,
|
||||
email,
|
||||
registration_date
|
||||
FROM core_dim_user
|
||||
WHERE registration_date >= today() - INTERVAL 30 DAY
|
||||
ORDER BY registration_date DESC;
|
||||
|
||||
/*
|
||||
### ✅ **Задание 6: Сравни данные с Greenplum**
|
||||
|
||||
1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
|
||||
2. Сравни с ClickHouse-результатами:
|
||||
- Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
|
||||
- Проверяй `NULL`, `0`, округления
|
||||
3. Напиши мини-отчёт: **"где совпадает, где нет, почему"
|
||||
|
||||
Совпадает ли SUM, AVG, MIN, MAX
|
||||
Совпадает и в гринплам, и в кликхаус
|
||||
|
||||
Проверяй NULL, 0, округления
|
||||
С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user