dmitry_medvedev/homework
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into OS-310

Browse Source
This commit is contained in:
maxim drozdov
2026-01-06 16:02:34 +04:00
parent 88db25d7d5 d514c04a88
commit 23e4bbf935
3 changed files with 815 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bhabibullin/clickhouse/clickhouse_pet.sql
+415
View File
@@ -0,0 +1,415 @@
--ПЕТ ПРОЕКТ СОЗДАНИЕ ТАБЛИЦ
CREATE TABLE bhabibullin.core_enrollments
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
course_id UInt32,
enrolled_at DateTime
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_enrollments',
'{replica}'
)
ORDER BY (user_id, enrolled_at);
CREATE TABLE bhabibullin.core_dim_user
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
name String,
age UInt32,
email String,
registration_date Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_dim_user',
'{replica}'
)
ORDER BY (user_id, registration_date);
CREATE TABLE bhabibullin.core_fact_lesson_views
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
lesson_id UInt32,
course_id UInt32,
viewed_at DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_fact_lesson_views',
'{replica}'
)
ORDER BY user_id;
CREATE TABLE bhabibullin.core_dim_course
ON CLUSTER clickhouse_cluster
(
course_id UInt32,
title String,
category String
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_dim_course',
'{replica}'
)
ORDER BY course_id;
drop table bhabibullin.core_dim_course ;
CREATE TABLE bhabibullin.core_dim_lesson
ON CLUSTER clickhouse_cluster
(
lesson_id UInt32,
title String,
duration_min UInt32,
course_id UInt32
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_dim_lesson',
'{replica}'
)
ORDER BY (lesson_id, course_id);
-- Этап 2. Создание витринных таблиц
-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
/*
- Спроектируй таблицу, где каждая строка — один урок.
- Для каждого урока должны храниться:
- ID и название урока
- ID и название курса, к которому он относится
- Общее число просмотров
- Количество уникальных пользователей, посмотревших урок
- Дата первого и последнего просмотра
*/
-- Основная таблица с AggregatingMergeTree
CREATE TABLE bhabibullin.lesson_popularity_summary
ON CLUSTER clickhouse_cluster
(
lesson_id UInt32,
lesson_title String,
course_id UInt32,
course_title String,
total_views AggregateFunction(sum, UInt64),
unique_users AggregateFunction(uniq, UInt32),
first_view AggregateFunction(min, DateTime64(3, 'UTC')),
last_view AggregateFunction(max, DateTime64(3, 'UTC'))
)
ENGINE = ReplicatedAggregatingMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary',
'{replica}'
)
ORDER BY (lesson_id, course_id)
SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW bhabibullin.mv_lesson_popularity_summary
ON CLUSTER clickhouse_cluster
TO bhabibullin.lesson_popularity_summary
AS
SELECT
flv.lesson_id AS lesson_id,
l.title AS lesson_title,
flv.course_id AS course_id,
c.title AS course_title,
sumState(1) AS total_views,
uniqState(flv.user_id) AS unique_users,
minState(flv.viewed_at) AS first_view,
maxState(flv.viewed_at) AS last_view
FROM core_fact_lesson_views AS flv
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN core_dim_course AS c ON flv.course_id = c.course_id
GROUP BY
flv.lesson_id,
l.title,
flv.course_id,
c.title;
INSERT INTO bhabibullin.lesson_popularity_summary
SELECT
flv.lesson_id,
l.title AS lesson_title,
flv.course_id,
c.title AS course_title,
count() AS total_views,
uniqExact(flv.user_id) AS unique_users,
min(flv.viewed_at) AS first_view,
max(flv.viewed_at) AS last_view
FROM core_fact_lesson_views AS flv
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN core_dim_course AS c ON flv.course_id = c.course_id
GROUP BY
flv.lesson_id,
l.title,
flv.course_id,
c.title;
select * from bhabibullin.mv_lesson_popularity_summary ;
select * from bhabibullin.lesson_popularity_summary ;
show tables;
/*
### ✅ **Задание 3: Создай витрину `inactive_users_summary`**
**Цель:** выявить пользователей, которые **зарегистрировались, но ничего не посмотрели**.
📌 **Что нужно сделать:**
- Подготовь таблицу, где каждая строка — один такой "неактивный" пользователь.
- Для каждого пользователя нужны:
- ID, имя, email, возраст
- Количество курсов, на которые он записался
- Дата регистрации
**Где пригодится:**
Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости.
*/
DROP TABLE bhabibullin.inactive_users_summary;
DROP VIEW bhabibullin.mv_inactive_users_summary ;
show tables;
CREATE TABLE bhabibullin.inactive_users_summary
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
name String,
email String,
age UInt32,
enrollment_count UInt32,
registration_date Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/inactive_users_summary',
'{replica}'
)
ORDER BY user_id;
CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary
ON CLUSTER clickhouse_cluster
TO bhabibullin.inactive_users_summary
AS
SELECT
cdu.user_id,
cdu.name,
cdu.email,
cdu.age,
countDistinct(flv.course_id) as enrollment_count,
cdu.registration_date
FROM core_dim_user cdu
LEFT ANTI JOIN core_fact_lesson_views flv
ON cdu.user_id = flv.user_id
GROUP BY
cdu.user_id,
cdu.name,
cdu.email,
cdu.age,
cdu.registration_date;
INSERT INTO bhabibullin.inactive_users_summary
SELECT
cdu.user_id,
cdu.name,
cdu.email,
cdu.age,
0 as enrollment_count,
cdu.registration_date
FROM core_dim_user cdu
WHERE (cdu.user_id, 1) NOT IN (
SELECT user_id, 1
FROM core_enrollments
GROUP BY user_id
);
select * from bhabibullin.inactive_users_summary ius ;
select * from bhabibullin.mv_inactive_users_summary mius ;
/*
### ✅ **Задание 4: Создай витрину `course_completion_rate`**
**Цель:** проанализировать, насколько хорошо пользователи проходят курсы.
📌 **Что нужно сделать:**
- Сформируй таблицу, где строка — это пользователь + курс.
- Для каждой пары нужно рассчитать:
- Общее число уроков в курсе
- Сколько из них просмотрел конкретный пользователь
- Коэффициент завершения (`lessons_viewed / lessons_in_course`)
*/
CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
user_name String,
course_id UInt32,
course_title String,
lessons_in_course UInt32,
lessons_viewed UInt32,
completion_rate Float64
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/course_completion_rate',
'{replica}'
)
ORDER BY (user_id, course_id);
CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate
ON CLUSTER clickhouse_cluster
TO bhabibullin.course_completion_rate
AS
WITH lessons_in_course AS (
SELECT
course_id,
count(Distinct lesson_id) AS lessons_in_course
FROM core_dim_lesson
GROUP BY course_id
),
views_user_course AS (
SELECT
user_id,
course_id,
count(Distinct lesson_id) AS lessons_viewed
FROM core_fact_lesson_views
GROUP BY user_id, course_id
)
SELECT
u.user_id AS user_id,
u.name AS user_name,
c.course_id AS course_id,
c.title AS course_title,
lic.lessons_in_course AS lessons_in_course,
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
ROUND(
COALESCE(
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
/ NULLIF(lic.lessons_in_course * 1.0, 0),
0
),
2
) AS completion_rate
FROM core_dim_user u
JOIN core_enrollments e ON u.user_id = e.user_id
JOIN core_dim_course c ON e.course_id = c.course_id
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id
ORDER BY
u.user_id,
c.course_id;
INSERT INTO bhabibullin.course_completion_rate
WITH lessons_in_course AS (
SELECT
course_id,
count(Distinct lesson_id) AS lessons_in_course
FROM core_dim_lesson
GROUP BY course_id
),
views_user_course AS (
SELECT
user_id,
course_id,
count(Distinct lesson_id) AS lessons_viewed
FROM core_fact_lesson_views
GROUP BY user_id, course_id
)
SELECT
u.user_id AS user_id,
u.name AS user_name,
c.course_id AS course_id,
c.title AS course_title,
lic.lessons_in_course AS lessons_in_course,
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
ROUND(
COALESCE(
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
/ NULLIF(lic.lessons_in_course * 1.0, 0),
0
),
2
) AS completion_rate
FROM core_dim_user u
JOIN core_enrollments e ON u.user_id = e.user_id
JOIN core_dim_course c ON e.course_id = c.course_id
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id;
select * from bhabibullin.course_completion_rate;
select * from bhabibullin.mv_course_completion_rate;
/*
# 📊 Этап 3. Анализ и запросы
### ✅ **Задание 5: Напиши SQL-запросы к витринам**
**Напиши свои вариации для практики.**
*/
--Топ-5 самых просматриваемых уроков
SELECT
lesson_title,
course_title,
total_views
FROM lesson_popularity_summary
ORDER BY total_views DESC
LIMIT 5;
--Неактивные пользователи, записавшиеся на курсы
SELECT
name,
email,
enrollment_count
FROM inactive_users_summary
WHERE enrollment_count > 0
ORDER BY registration_date DESC;
--Курсы с самым высоким процентом завершения
SELECT
course_title,
AVG(completion_rate) AS avg_completion
FROM course_completion_rate
GROUP BY course_title
ORDER BY avg_completion DESC
LIMIT 5;
-- топ 5 курсов по количеству уникальных пользователей
SELECT
course_title,
unique_users
FROM lesson_popularity_summary
ORDER BY unique_users DESC
LIMIT 5;
--новые пользователи за прошлый месяц
SELECT
user_id,
name,
email,
registration_date
FROM core_dim_user
WHERE registration_date >= today() - INTERVAL 30 DAY
ORDER BY registration_date DESC;
/*
### ✅ **Задание 6: Сравни данные с Greenplum**
1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
2. Сравни с ClickHouse-результатами:
- Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
- Проверяй `NULL`, `0`, округления
3. Напиши мини-отчёт: **"где совпадает, где нет, почему"
Совпадает ли SUM, AVG, MIN, MAX
Совпадает и в гринплам, и в кликхаус
Проверяй NULL, 0, округления
С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения
*/
bhabibullin/greenplum/pet_project_gp.sql
+218
View File
@@ -0,0 +1,218 @@
CREATE SCHEMA raw;
CREATE SCHEMA core;
CREATE SCHEMA dm;
CREATE TABLE raw.users (
id INT,
name VARCHAR(255),
age INT,
email VARCHAR(255),
registration_date DATE
);
CREATE TABLE raw.courses (
id INT,
title VARCHAR(255),
category VARCHAR(255),
created_at DATE
);
CREATE TABLE raw.lessons (
id INT,
course_id INT,
title VARCHAR(255),
duration_min INT
);
CREATE TABLE raw.enrollments (
id INT,
user_id INT,
course_id INT,
enrolled_at DATE
);
CREATE TABLE raw.lesson_views (
id INT,
user_id INT,
lesson_id INT,
viewed_at TIMESTAMP
);
CREATE TABLE core.dim_user (
user_id INT PRIMARY KEY,
name TEXT,
age INT,
email TEXT,
registration_date DATE
);
CREATE TABLE core.dim_course (
course_id INT PRIMARY KEY,
title TEXT,
category TEXT
);
CREATE TABLE core.dim_lesson (
lesson_id INT PRIMARY KEY,
title TEXT,
duration_min INT,
course_id INT REFERENCES core.dim_course(course_id)
);
CREATE TABLE core.fact_lesson_views (
user_id INT REFERENCES core.dim_user(user_id),
lesson_id INT REFERENCES core.dim_lesson(lesson_id),
course_id INT REFERENCES core.dim_course(course_id),
viewed_at TIMESTAMP
);
CREATE TABLE core.enrollments (
user_id INT REFERENCES core.dim_user(user_id),
course_id INT REFERENCES core.dim_course(course_id),
enrolled_at TIMESTAMP
);
INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
SELECT DISTINCT
id::INT,
name::TEXT,
age::INT,
email::TEXT,
registration_date::DATE
FROM raw.users;
INSERT INTO core.dim_course (course_id, title, category)
SELECT DISTINCT
id::INT,
title::TEXT,
category::TEXT
FROM raw.courses;
INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
SELECT DISTINCT
id::INT,
title::TEXT,
duration_min::INT,
course_id::INT
FROM raw.lessons;
INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
SELECT DISTINCT
rw.user_id::INT,
rw.lesson_id::INT,
l.course_id::INT,
rw.viewed_at::TIMESTAMP
FROM raw.lesson_views rw
JOIN raw.lessons l ON rw.lesson_id = l.id;
INSERT INTO core.enrollments (user_id, course_id, enrolled_at)
SELECT DISTINCT
user_id::INT,
course_id::INT,
enrolled_at::DATE
FROM raw.enrollments;
CREATE MATERIALIZED VIEW dm.dm_lesson_popularity_summary AS
SELECT
dl.lesson_id,
dl.title AS lesson_title,
dc.course_id,
dc.title AS course_title,
COUNT(*) AS total_views,
COUNT(DISTINCT flv.user_id) AS unique_users,
MIN(flv.viewed_at) AS first_view,
MAX(flv.viewed_at) AS last_view
FROM core.dim_lesson dl
JOIN core.dim_course dc ON dc.course_id = dl.course_id
JOIN core.fact_lesson_views flv ON flv.lesson_id = dl.lesson_id
GROUP BY dl.lesson_id, dl.title, dc.course_id, dc.title;
/*
Цель:
Показать популярность каждого урока.
Поля витрины:
- lesson_id — идентификатор урока
- lesson_title — название урока
- course_id — идентификатор курса
- course_title — название курса
- total_views — общее число просмотров урока
- unique_users — количество уникальных пользователей
- first_view — дата первого просмотра
- last_view — дата последнего просмотра
*/
CREATE MATERIALIZED VIEW dm.dm_inactive_users_summary AS
SELECT
du.user_id,
du.name AS user_name,
du.email,
du.age,
COUNT(DISTINCT e.course_id) AS enrollments_count,
du.registration_date
FROM core.dim_user du
LEFT JOIN core.enrollments e ON e.user_id = du.user_id
WHERE NOT EXISTS (
SELECT 1
FROM core.fact_lesson_views flv
WHERE flv.user_id = du.user_id
)
GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date;
/*
Цель:
Определить невовлечённых пользователей и потенциальный отток.
Поля витрины:
- user_id — идентификатор пользователя
- user_name — имя
- email — почта
- age — возраст
- enrollments_count — количество курсов
- registration_date — дата регистрации
*/
CREATE MATERIALIZED VIEW dm.course_completion_rate AS
SELECT
du.user_id,
du.name AS user_name,
c.course_id,
c.title AS course_title,
COUNT(DISTINCT dl.lesson_id) AS lessons_in_course,
COUNT(DISTINCT flv.lesson_id) AS lessons_viewed,
ROUND(
COUNT(DISTINCT flv.lesson_id) * 1.0 /
NULLIF(COUNT(DISTINCT dl.lesson_id), 0),
2
) AS completion_rate
FROM core.dim_user du
JOIN core.enrollments e ON du.user_id = e.user_id
JOIN core.dim_course c ON e.course_id = c.course_id
JOIN core.dim_lesson dl ON c.course_id = dl.course_id
LEFT JOIN core.fact_lesson_views flv ON dl.lesson_id = flv.lesson_id
AND flv.user_id = du.user_id
GROUP BY du.user_id, du.name, c.course_id, c.title
ORDER BY du.user_id, c.course_id;
/*
Цель:
Анализировать, насколько пользователи проходят курсы до конца.
Поля витрины:
- user_id — идентификатор пользователя
- user_name — имя пользователя
- course_id — идентификатор курса
- course_title — название курса
- total_lessons — количество уроков в курсе
- lessons_viewed — число просмотренных уроков
- completion_rate — lessons_viewed / total_lessons
*/
bhabibullin/postgre/postrge_pet_project.sql
+182
View File
@@ -0,0 +1,182 @@
create table users(
id int primary key,
name varchar(255),
age int,
email varchar(255),
registration_date date
);
insert into users(id, name, age, email, registration_date) values
(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'),
(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'),
(3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'),
(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'),
(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10');
create table courses(
id int primary key,
title varchar(255),
category varchar(255),
created_at date
);
insert into courses(id, title, category, created_at) values
(1, 'SQL для начинающих', 'data', '2023-01-01'),
(2, 'Python для анализа данных', 'programming', '2023-01-15'),
(3, 'BI с нуля', 'business', '2023-03-01');
create table lessons(
id int primary key,
course_id int,
title varchar(255),
duration_min int,
foreign key (course_id) references courses(id)
);
insert into lessons(id, course_id, title, duration_min) values
(1, 1, 'SELECT и FROM', 10),
(2, 1, 'JOIN', 15),
(3, 2, 'Pandas', 20),
(4, 3, 'Основы BI', 12),
(5, 3, 'Метрики и дашборды', 18);
create table enrollments(
id int primary key,
user_id int,
course_id int,
enrolled_at date,
foreign key (user_id) references users(id),
foreign key (course_id) references courses(id)
);
insert into enrollments (id, user_id, course_id, enrolled_at) values
(1, 1, 1, '2023-01-15'),
(2, 1, 2, '2023-02-01'),
(3, 2, 1, '2023-01-20'),
(4, 3, 2, '2023-03-05'),
(5, 4, 3, '2023-04-01');
create table lesson_views(
id int primary key,
user_id int,
lesson_id int,
viewed_at timestamp,
foreign key (user_id) references users(id),
foreign key (lesson_id) references lessons(id)
);
insert into lesson_views(id, user_id, lesson_id, viewed_at) values
(1, 1, 1, '2023-01-16 10:00:00'),
(2, 1, 2, '2023-01-16 10:15:00'),
(3, 2, 1, '2023-01-21 09:00:00'),
(4, 3, 3, '2023-03-06 12:00:00'),
(5, 4, 4, '2023-04-02 10:00:00'),
(6, 4, 5, '2023-04-03 10:00:00');
create materialized view user_activity_summary as (
with available_lessons as (
select
e.user_id,
count(distinct l.id) as cnt_lesson
from lessons l
join enrollments e on e.course_id = l.course_id
group by e.user_id
)
select
u.id,
u.name,
count(distinct e.course_id) as enrrollment_count,
count(distinct lv.lesson_id) as viewed_lessons_count,
max(lv.viewed_at) as last_view,
min(lv.viewed_at) as first_view,
round(count(distinct lv.lesson_id) * 100.0 / NULLIF(al.cnt_lesson, 0), 4) as engagement_rate
from users u
join enrollments e on u.id = e.user_id
left join lesson_views lv on lv.user_id = u.id
join available_lessons al on al.user_id = u.id
group by u.id, u.name, al.cnt_lesson
);
/*
Витрина user_activity_summary
Цель - показать аналитику вовлеченности пользователей в обучение на платформе
user_id - идентификатор пользователя
name - имя пользователя
enrollment_count - количество записей на курсы
viewed_lessons_count - количество просмотренных уроков
last_view - последний просмотр
first_view - первый просмотр
engagement_rate - вовлеченность
*/
create materialized view course_summary as(
with view_cnt as (
select l.course_id,
count(lv.lesson_id) cnt
from lessons l
join lesson_views lv on l.id = lv.lesson_id
group by l.course_id
)
select
c.id as course_id,
c.title,
count(distinct e.user_id) unique_users,
count(distinct l.id) lessons_count,
vc.cnt total_views,
round(vc.cnt * 1.0 / count(distinct e.user_id), 2) as avg_views_per_user,
min(lv.viewed_at) as first_viewed_at,
max(lv.viewed_at) as last_viewed_at
from
courses c
join lessons l on c.id = l.course_id
join enrollments e on e.course_id = c.id
join view_cnt vc on vc.course_id = c.id
left join lesson_views lv on lv.lesson_id = l.id
group by c.title, vc.cnt, c.id
order by course_id
);
/*
Витрина course_summary
Цель - показать анализ эффективности по каждому курсу
course_id - идентификатор курса
course title - название курса
unique_users - кол-во уникальных пользователей в рамках одного курса
lessons_count - кол-во уроков
total_views - общее число просмотров уроков по курсу
avg_views_per_user - total_views / unique_users Среднее число просмотров на пользователя, записанного на курс
first_viewed_at - дата первого просмотра
last_viewed_at - дата последнего просмотра
*/
create materialized view platform_summary as (
select
count(distinct u.id) as total_users,
count(distinct c.id) as total_courses,
count(distinct l.id) as total_lessons,
count(distinct lv.user_id) as users_with_views,
round(count(distinct l.id) * 1.0 / NULLIF(count(distinct c.id), 0), 2) as avg_lessons_per_course,
count(lv.lesson_id) * 1.0 / NULLIF(count(distinct l.id), 0) as avg_lessons_per_lesson
from users u
left join lesson_views lv on lv.user_id = u.id
left join lessons l on l.id = lv.lesson_id
left join courses c on c.id = l.course_id
);
/*
Витрина platform_summary
Цель - показать общую отчетность по взаимодействию
total_users - всего пользователей
total_courses - всего курсов
total_lessons - всего уроков
users_with_views - сколько пользователей с минимум 1 просмотром
avg_lessons_per_course - среднее число уроков на курс
avg_views_per_lesson - среднее число просмотров на урок
*/