From 2f80be77edc7c9b566bc84af429555c6429bdea1 Mon Sep 17 00:00:00 2001 From: Batirbulat Date: Tue, 9 Dec 2025 20:02:19 +0300 Subject: [PATCH 1/6] pet gp postgre done --- bhabibullin/greenplum/pet_project_gp.sql | 180 ++++++++++++++++++++ bhabibullin/postgre/postrge_pet_project.sql | 143 ++++++++++++++++ 2 files changed, 323 insertions(+) create mode 100644 bhabibullin/greenplum/pet_project_gp.sql create mode 100644 bhabibullin/postgre/postrge_pet_project.sql diff --git a/bhabibullin/greenplum/pet_project_gp.sql b/bhabibullin/greenplum/pet_project_gp.sql new file mode 100644 index 0000000..3481821 --- /dev/null +++ b/bhabibullin/greenplum/pet_project_gp.sql @@ -0,0 +1,180 @@ +CREATE SCHEMA raw; +CREATE SCHEMA core; +CREATE SCHEMA dm; + +CREATE TABLE raw.users ( + id INT, + name VARCHAR(255), + age INT, + email VARCHAR(255), + registration_date DATE +); + +CREATE TABLE raw.courses ( + id INT, + title VARCHAR(255), + category VARCHAR(255), + created_at DATE +); + +CREATE TABLE raw.lessons ( + id INT, + course_id INT, + title VARCHAR(255), + duration_min INT +); + +CREATE TABLE raw.enrollments ( + id INT, + user_id INT, + course_id INT, + enrolled_at DATE +); + +CREATE TABLE raw.lesson_views ( + id INT, + user_id INT, + lesson_id INT, + viewed_at TIMESTAMP +); + +CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE +); + +CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT +); + +CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) +); + +CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP +); + +CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP +); + +INSERT INTO core.dim_user (user_id, name, age, email, registration_date) +SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE +FROM raw.users; + +INSERT INTO core.dim_course (course_id, title, category) +SELECT DISTINCT + id::INT, + title::TEXT, + category::TEXT +FROM raw.courses; + +INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) +SELECT DISTINCT + id::INT, + title::TEXT, + duration_min::INT, + course_id::INT +FROM raw.lessons; + +INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) +SELECT DISTINCT + rw.user_id::INT, + rw.lesson_id::INT, + l.course_id::INT, + rw.viewed_at::TIMESTAMP +FROM raw.lesson_views rw +JOIN raw.lessons l ON rw.lesson_id = l.id; + + +INSERT INTO core.enrollments (user_id, course_id, enrolled_at) +SELECT DISTINCT + user_id::INT, + course_id::INT, + enrolled_at::DATE +FROM raw.enrollments; + +UPDATE core.enrollments e +SET enrolled_at = lv.viewed_at +FROM raw.lesson_views lv +WHERE lv.user_id = e.user_id; + +ALTER TABLE core.enrollments +ALTER COLUMN enrolled_at TYPE TIMESTAMP; + +CREATE MATERIALIZED VIEW dm.dm_lesson_popularity_summary AS +SELECT + dl.lesson_id, + dl.title AS lesson_title, + dc.course_id, + dc.title AS course_title, + COUNT(flv.lesson_id) AS total_views, + COUNT(DISTINCT du.user_id) AS unique_users, + MIN(flv.viewed_at) AS first_view, + MAX(flv.viewed_at) AS last_view +FROM core.dim_lesson dl +JOIN core.dim_course dc ON dc.course_id = dl.course_id +JOIN core.fact_lesson_views flv ON flv.lesson_id = dl.lesson_id +JOIN core.dim_user du ON du.user_id = flv.user_id +GROUP BY dl.lesson_id, dl.title, dc.course_id, dc.title +ORDER BY dl.lesson_id; + +select * from dm.dm_lesson_popularity_summary +order by lesson_id; + +CREATE MATERIALIZED VIEW dm.dm_inactive_users_summary AS +SELECT + du.user_id, + du.name AS user_name, + du.email, + du.age, + COUNT(e.course_id) AS enrollments_count, + du.registration_date +FROM core.dim_user du +LEFT JOIN core.enrollments e ON e.user_id = du.user_id +LEFT JOIN core.fact_lesson_views flv ON flv.user_id = du.user_id +WHERE flv.user_id IS NULL +GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date; + + +CREATE MATERIALIZED VIEW dm.course_completion_rate AS +SELECT + du.user_id, + du.name AS user_name, + c.course_id, + c.title AS course_title, + COUNT(DISTINCT dl.lesson_id) AS lessons_in_course, + COUNT(DISTINCT flv.lesson_id) AS lessons_viewed, + ROUND( + COUNT(DISTINCT flv.lesson_id) * 1.0 / + NULLIF(COUNT(DISTINCT dl.lesson_id), 0), + 2 + ) AS completion_rate +FROM core.dim_user du +JOIN core.enrollments e ON du.user_id = e.user_id +JOIN core.dim_course c ON e.course_id = c.course_id +JOIN core.dim_lesson dl ON c.course_id = dl.course_id +LEFT JOIN core.fact_lesson_views flv ON dl.lesson_id = flv.lesson_id + AND flv.user_id = du.user_id +GROUP BY du.user_id, du.name, c.course_id, c.title +ORDER BY du.user_id, c.course_id; + diff --git a/bhabibullin/postgre/postrge_pet_project.sql b/bhabibullin/postgre/postrge_pet_project.sql new file mode 100644 index 0000000..2a7fda1 --- /dev/null +++ b/bhabibullin/postgre/postrge_pet_project.sql @@ -0,0 +1,143 @@ +create table users( + id int primary key, + name varchar(255), + age int, + email varchar(255), + registration_date date +); + +insert into users(id, name, age, email, registration_date) values +(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'), +(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'), +(3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'), +(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'), +(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10'); + +create table courses( + id int primary key, + title varchar(255), + category varchar(255), + created_at date +); + +insert into courses(id, title, category, created_at) values +(1, 'SQL для начинающих', 'data', '2023-01-01'), +(2, 'Python для анализа данных', 'programming', '2023-01-15'), +(3, 'BI с нуля', 'business', '2023-03-01'); + +create table lessons( + id int primary key, + course_id int, + title varchar(255), + duration_min int, + foreign key (course_id) references courses(id) +); + +insert into lessons(id, course_id, title, duration_min) values +(1, 1, 'SELECT и FROM', 10), +(2, 1, 'JOIN', 15), +(3, 2, 'Pandas', 20), +(4, 3, 'Основы BI', 12), +(5, 3, 'Метрики и дашборды', 18); + +create table enrollments( + id int primary key, + user_id int, + course_id int, + enrolled_at date, + foreign key (user_id) references users(id), + foreign key (course_id) references courses(id) +); + +insert into enrollments (id, user_id, course_id, enrolled_at) values +(1, 1, 1, '2023-01-15'), +(2, 1, 2, '2023-02-01'), +(3, 2, 1, '2023-01-20'), +(4, 3, 2, '2023-03-05'), +(5, 4, 3, '2023-04-01'); + +create table lesson_views( + id int primary key, + user_id int, + lesson_id int, + viewed_at timestamp, + foreign key (user_id) references users(id), + foreign key (lesson_id) references lessons(id) +); + +insert into lesson_views(id, user_id, lesson_id, viewed_at) values +(1, 1, 1, '2023-01-16 10:00:00'), +(2, 1, 2, '2023-01-16 10:15:00'), +(3, 2, 1, '2023-01-21 09:00:00'), +(4, 3, 3, '2023-03-06 12:00:00'), +(5, 4, 4, '2023-04-02 10:00:00'), +(6, 4, 5, '2023-04-03 10:00:00'); + + +create materialized view user_activity_summary as ( +with available_lessons as ( + select + e.user_id, + count(distinct l.id) as cnt_lesson + from lessons l + join enrollments e on e.course_id = l.course_id + group by e.user_id +) +select + u.id, + u.name, + count(distinct e.course_id) as enrrollment_count, + count(distinct lv.lesson_id) as viewed_lessons_count, + max(lv.viewed_at) as last_view, + min(lv.viewed_at) as first_view, + round( + count(distinct lv.lesson_id) * 100.0 / al.cnt_lesson + ,2) || '%' as engagegement_rate +from users u +join enrollments e on u.id = e.user_id +join lesson_views lv on lv.user_id = u.id +join available_lessons al on al.user_id = u.id +group by u.id, u.name, al.cnt_lesson +); + +create materialized view course_summary as( +with view_cnt as ( + select l.course_id, + count(lv.lesson_id) cnt + from lessons l + join lesson_views lv on l.id = lv.lesson_id + group by l.course_id +) +select + c.id as course_id, + c.title, + count(distinct e.user_id) unique_users, + count(distinct l.id) lessons_count, + vc.cnt total_views, + round(vc.cnt * 1.0 / count(distinct e.user_id), 2) as avg_views_per_user, + min(lv.viewed_at) as first_viewed_at, + max(lv.viewed_at) as last_viewed_at +from + courses c + join lessons l on c.id = l.course_id + join enrollments e on e.course_id = c.id + join view_cnt vc on vc.course_id = c.id + join lesson_views lv on lv.lesson_id = l.id +group by c.title, vc.cnt, c.id +order by course_id +); + +create materialized view platform_summary as ( +select + count(distinct u.id) as total_users, + count(distinct c.id) as total_courses, + count(distinct l.id) as total_lessons, + count(distinct lv.user_id) as users_with_views, + round(count(distinct l.id) * 1.0 / count(distinct c.id), 2) as avg_lessons_per_course, + count(lv.lesson_id) * 1.0 / count(distinct l.id) as avg_lessons_per_lesson +from users u +left join lesson_views lv on lv.user_id = u.id +left join lessons l on l.id = lv.lesson_id +left join courses c on c.id = l.course_id +); + From a1421a8fa01bc0c862550caca495189efac9b614 Mon Sep 17 00:00:00 2001 From: Batirbulat Date: Tue, 9 Dec 2025 20:05:36 +0300 Subject: [PATCH 2/6] --- bhabibullin/greenplum/pet_project_gp.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bhabibullin/greenplum/pet_project_gp.sql b/bhabibullin/greenplum/pet_project_gp.sql index 3481821..3d7ae2f 100644 --- a/bhabibullin/greenplum/pet_project_gp.sql +++ b/bhabibullin/greenplum/pet_project_gp.sql @@ -78,7 +78,7 @@ SELECT DISTINCT name::TEXT, age::INT, email::TEXT, - registration_date::DATE + registration_date::DATE FROM raw.users; INSERT INTO core.dim_course (course_id, title, category) From 7fb8409941d707028f870693154b9c1b7c3fb2a7 Mon Sep 17 00:00:00 2001 From: Batirbulat Date: Wed, 17 Dec 2025 19:31:06 +0300 Subject: [PATCH 3/6] =?UTF-8?q?=D0=9D=D0=B5=20=D0=BC=D0=B5=D0=BD=D1=8F?= =?UTF-8?q?=D0=BB=20=D0=BD=D0=B8=D1=87=D0=B5=D0=B3=D0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bhabibullin/clickhouse/clickhouse_pet.sql | 384 ++++++++++++++++++++ bhabibullin/greenplum/pet_project_gp.sql | 57 ++- bhabibullin/postgre/postrge_pet_project.sql | 41 +++ 3 files changed, 472 insertions(+), 10 deletions(-) create mode 100644 bhabibullin/clickhouse/clickhouse_pet.sql diff --git a/bhabibullin/clickhouse/clickhouse_pet.sql b/bhabibullin/clickhouse/clickhouse_pet.sql new file mode 100644 index 0000000..68bd351 --- /dev/null +++ b/bhabibullin/clickhouse/clickhouse_pet.sql @@ -0,0 +1,384 @@ +--ПЕТ ПРОЕКТ СОЗДАНИЕ ТАБЛИЦ + +CREATE TABLE bhabibullin.core_enrollments +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + course_id UInt32, + enrolled_at DateTime +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_enrollments', + '{replica}' +) +ORDER BY (user_id, enrolled_at); + + +CREATE TABLE bhabibullin.core_dim_user +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + name String, + age UInt32, + email String, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_dim_user', + '{replica}' +) +ORDER BY (user_id, registration_date); + + +CREATE TABLE bhabibullin.core_fact_lesson_views +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + lesson_id UInt32, + course_id UInt32, + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_fact_lesson_views', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE bhabibullin.core_dim_course +ON CLUSTER clickhouse_cluster +( + course_id UInt32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_dim_course', + '{replica}' +) +ORDER BY course_id; + +drop table bhabibullin.core_dim_course ; + +CREATE TABLE bhabibullin.core_dim_lesson +ON CLUSTER clickhouse_cluster +( + lesson_id UInt32, + title String, + duration_min UInt32, + course_id UInt32 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_dim_lesson', + '{replica}' +) +ORDER BY (lesson_id, course_id); + + +-- Этап 2. Создание витринных таблиц +-- Задание 2: Подготовь структуру витрины lesson_popularity_summary +/* +- Спроектируй таблицу, где каждая строка — один урок. +- Для каждого урока должны храниться: + - ID и название урока + - ID и название курса, к которому он относится + - Общее число просмотров + - Количество уникальных пользователей, посмотревших урок + - Дата первого и последнего просмотра + */ + +CREATE TABLE bhabibullin.lesson_popularity_summary +( + lesson_id UInt32, + lesson_title String, + course_id UInt32, + course_title String, + total_views UInt32, + unique_users UInt32, + first_view DateTime64(3, 'UTC'), + last_view DateTime64(3, 'UTC') +) +ENGINE = MergeTree +ORDER BY (lesson_id, course_id); + +CREATE MATERIALIZED VIEW mv_lesson_popularity_summary +TO bhabibullin.lesson_popularity_summary +AS +SELECT + flv.lesson_id AS lesson_id, + l.title AS lesson_title, + flv.course_id AS course_id, + c.title AS course_title, + count() AS total_views, + uniqExact(flv.user_id) AS unique_users, + min(flv.viewed_at) AS first_view, + max(flv.viewed_at) AS last_view +FROM core_fact_lesson_views AS flv +JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN core_dim_course AS c ON flv.course_id = c.course_id +GROUP BY + flv.lesson_id, + l.title, + flv.course_id, + c.title; + +INSERT INTO bhabibullin.lesson_popularity_summary +SELECT + flv.lesson_id, + l.title AS lesson_title, + flv.course_id, + c.title AS course_title, + count() AS total_views, + uniqExact(flv.user_id) AS unique_users, + min(flv.viewed_at) AS first_view, + max(flv.viewed_at) AS last_view +FROM core_fact_lesson_views AS flv +JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN core_dim_course AS c ON flv.course_id = c.course_id +GROUP BY + flv.lesson_id, + l.title, + flv.course_id, + c.title; + +select * from bhabibullin.mv_lesson_popularity_summary mlps ; +select * from bhabibullin.lesson_popularity_summary lps ; + +/* + ### ✅ **Задание 3: Создай витрину `inactive_users_summary`** + +**Цель:** выявить пользователей, которые **зарегистрировались, но ничего не посмотрели**. + +📌 **Что нужно сделать:** + +- Подготовь таблицу, где каждая строка — один такой "неактивный" пользователь. +- Для каждого пользователя нужны: + - ID, имя, email, возраст + - Количество курсов, на которые он записался + - Дата регистрации + +**Где пригодится:** + +Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости. + */ + +CREATE TABLE bhabibullin.inactive_users_summary( + user_id UInt32, + name String, + email String, + age UInt32, + enrollment_count UInt32, + registration_date Date +) +ENGINE = MergeTree() +ORDER BY user_id; + +CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary +to bhabibullin.inactive_users_summary +AS +SELECT + cdu.user_id, + cdu.name, + cdu.email, + cdu.age, + 0 as enrollment_count, + cdu.registration_date +FROM core_dim_user cdu +WHERE (cdu.user_id, 1) NOT IN ( + SELECT user_id, 1 + FROM core_enrollments + GROUP BY user_id +); + +INSERT INTO bhabibullin.inactive_users_summary +SELECT + cdu.user_id, + cdu.name, + cdu.email, + cdu.age, + 0 as enrollment_count, + cdu.registration_date +FROM core_dim_user cdu +WHERE (cdu.user_id, 1) NOT IN ( + SELECT user_id, 1 + FROM core_enrollments + GROUP BY user_id +); + +/* + ### ✅ **Задание 4: Создай витрину `course_completion_rate`** + +**Цель:** проанализировать, насколько хорошо пользователи проходят курсы. + +📌 **Что нужно сделать:** + +- Сформируй таблицу, где строка — это пользователь + курс. +- Для каждой пары нужно рассчитать: + - Общее число уроков в курсе + - Сколько из них просмотрел конкретный пользователь + - Коэффициент завершения (`lessons_viewed / lessons_in_course`) + */ + + +CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate +( + user_id UInt32, + user_name String, + course_id UInt32, + course_title String, + lessons_in_course UInt32, + lessons_viewed UInt32, + completion_rate Float64 +) +ENGINE = MergeTree() +ORDER BY (user_id, course_id); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate +TO bhabibullin.course_completion_rate +AS +WITH lessons_in_course AS ( + SELECT + course_id, + count(Distinct lesson_id) AS lessons_in_course + FROM core_dim_lesson + GROUP BY course_id +), +views_user_course AS ( + SELECT + user_id, + course_id, + count(Distinct lesson_id) AS lessons_viewed + FROM core_fact_lesson_views + GROUP BY user_id, course_id +) +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + lic.lessons_in_course AS lessons_in_course, + coalesce(vuc.lessons_viewed, 0) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0) + / NULLIF(lic.lessons_in_course * 1.0, 0), + 0 + ), + 2 + ) AS completion_rate +FROM core_dim_user u +JOIN core_enrollments e ON u.user_id = e.user_id +JOIN core_dim_course c ON e.course_id = c.course_id +JOIN lessons_in_course AS lic ON c.course_id = lic.course_id +LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id +ORDER BY + u.user_id, + c.course_id; + +INSERT INTO bhabibullin.course_completion_rate +WITH lessons_in_course AS ( + SELECT + course_id, + count(Distinct lesson_id) AS lessons_in_course + FROM core_dim_lesson + GROUP BY course_id +), +views_user_course AS ( + SELECT + user_id, + course_id, + count(Distinct lesson_id) AS lessons_viewed + FROM core_fact_lesson_views + GROUP BY user_id, course_id +) +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + lic.lessons_in_course AS lessons_in_course, + coalesce(vuc.lessons_viewed, 0) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0) + / NULLIF(lic.lessons_in_course * 1.0, 0), + 0 + ), + 2 + ) AS completion_rate +FROM core_dim_user u +JOIN core_enrollments e ON u.user_id = e.user_id +JOIN core_dim_course c ON e.course_id = c.course_id +JOIN lessons_in_course AS lic ON c.course_id = lic.course_id +LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id; + +/* + # 📊 Этап 3. Анализ и запросы + +### ✅ **Задание 5: Напиши SQL-запросы к витринам** + +**Напиши свои вариации для практики.** + */ + +--Топ-5 самых просматриваемых уроков +SELECT + lesson_title, + course_title, + total_views +FROM lesson_popularity_summary +ORDER BY total_views DESC +LIMIT 5; + +--Неактивные пользователи, записавшиеся на курсы +SELECT + name, + email, + registered_courses_count +FROM inactive_users_summary +WHERE registered_courses_count > 0 +ORDER BY registration_date DESC; + +--Курсы с самым высоким процентом завершения +SELECT + course_title, + AVG(completion_rate) AS avg_completion +FROM course_completion_rate +GROUP BY course_title +ORDER BY avg_completion DESC +LIMIT 5; + +-- топ 5 курсов по количеству уникальных пользователей +SELECT + course_title, + uniqExact(user_id) AS active_users +FROM lesson_popularity_summary +GROUP BY course_title +ORDER BY active_users DESC +LIMIT 5; + +--новые пользователи за прошлый месяц +SELECT + user_id, + name, + email, + registration_date +FROM core_dim_user +WHERE registration_date >= today() - INTERVAL 30 DAY +ORDER BY registration_date DESC; + +/* + ### ✅ **Задание 6: Сравни данные с Greenplum** + +1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum +2. Сравни с ClickHouse-результатами: + - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX` + - Проверяй `NULL`, `0`, округления +3. Напиши мини-отчёт: **"где совпадает, где нет, почему" + +Совпадает ли SUM, AVG, MIN, MAX +Совпадает и в гринплам, и в кликхаус + +Проверяй NULL, 0, округления +С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения +*/ + diff --git a/bhabibullin/greenplum/pet_project_gp.sql b/bhabibullin/greenplum/pet_project_gp.sql index 3d7ae2f..d4cacd5 100644 --- a/bhabibullin/greenplum/pet_project_gp.sql +++ b/bhabibullin/greenplum/pet_project_gp.sql @@ -113,14 +113,6 @@ SELECT DISTINCT enrolled_at::DATE FROM raw.enrollments; -UPDATE core.enrollments e -SET enrolled_at = lv.viewed_at -FROM raw.lesson_views lv -WHERE lv.user_id = e.user_id; - -ALTER TABLE core.enrollments -ALTER COLUMN enrolled_at TYPE TIMESTAMP; - CREATE MATERIALIZED VIEW dm.dm_lesson_popularity_summary AS SELECT dl.lesson_id, @@ -138,8 +130,24 @@ JOIN core.dim_user du ON du.user_id = flv.user_id GROUP BY dl.lesson_id, dl.title, dc.course_id, dc.title ORDER BY dl.lesson_id; -select * from dm.dm_lesson_popularity_summary -order by lesson_id; + +/* + + Цель: + Показать популярность каждого урока. + + Поля витрины: + - lesson_id — идентификатор урока + - lesson_title — название урока + - course_id — идентификатор курса + - course_title — название курса + - total_views — общее число просмотров урока + - unique_users — количество уникальных пользователей + - first_view — дата первого просмотра + - last_view — дата последнего просмотра + +*/ + CREATE MATERIALIZED VIEW dm.dm_inactive_users_summary AS SELECT @@ -155,6 +163,20 @@ LEFT JOIN core.fact_lesson_views flv ON flv.user_id = du.user_id WHERE flv.user_id IS NULL GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date; +/* + +Цель: + Определить невовлечённых пользователей и потенциальный отток. + + Поля витрины: + - user_id — идентификатор пользователя + - user_name — имя + - email — почта + - age — возраст + - enrollments_count — количество курсов + - registration_date — дата регистрации + +*/ CREATE MATERIALIZED VIEW dm.course_completion_rate AS SELECT @@ -178,3 +200,18 @@ LEFT JOIN core.fact_lesson_views flv ON dl.lesson_id = flv.lesson_id GROUP BY du.user_id, du.name, c.course_id, c.title ORDER BY du.user_id, c.course_id; +/* + + Цель: + Анализировать, насколько пользователи проходят курсы до конца. + + Поля витрины: + - user_id — идентификатор пользователя + - user_name — имя пользователя + - course_id — идентификатор курса + - course_title — название курса + - total_lessons — количество уроков в курсе + - lessons_viewed — число просмотренных уроков + - completion_rate — lessons_viewed / total_lessons + +*/ \ No newline at end of file diff --git a/bhabibullin/postgre/postrge_pet_project.sql b/bhabibullin/postgre/postrge_pet_project.sql index 2a7fda1..fa642d2 100644 --- a/bhabibullin/postgre/postrge_pet_project.sql +++ b/bhabibullin/postgre/postrge_pet_project.sql @@ -100,6 +100,20 @@ join available_lessons al on al.user_id = u.id group by u.id, u.name, al.cnt_lesson ); +/* + Витрина user_activity_summary + Цель - показать аналитику вовлеченности пользователей в обучение на платформе + + user_id - идентификатор пользователя + name - имя пользователя + enrollment_count - количество записей на курсы + viewed_lessons_count - количество просмотренных уроков + last_view - последний просмотр + first_view - первый просмотр + engagement_rate - вовлеченность + +*/ + create materialized view course_summary as( with view_cnt as ( select l.course_id, @@ -127,6 +141,21 @@ group by c.title, vc.cnt, c.id order by course_id ); +/* + Витрина course_summary + Цель - показать анализ эффективности по каждому курсу + + course_id - идентификатор курса + course title - название курса + unique_users - кол-во уникальных пользователей в рамках одного курса + lessons_count - кол-во уроков + total_views - общее число просмотров уроков по курсу + avg_views_per_user - total_views / unique_users Среднее число просмотров на пользователя, записанного на курс + first_viewed_at - дата первого просмотра + last_viewed_at - дата последнего просмотра +*/ + + create materialized view platform_summary as ( select count(distinct u.id) as total_users, @@ -141,3 +170,15 @@ left join lessons l on l.id = lv.lesson_id left join courses c on c.id = l.course_id ); +/* + Витрина platform_summary + Цель - показать общую отчетность по взаимодействию + + total_users - всего пользователей + total_courses - всего курсов + total_lessons - всего уроков + users_with_views - сколько пользователей с минимум 1 просмотром + avg_lessons_per_course - среднее число уроков на курс + avg_views_per_lesson - среднее число просмотров на урок + +*/ \ No newline at end of file From ad47dd718e31c0bb3b4ee4f6480e13ceb3a1620e Mon Sep 17 00:00:00 2001 From: Batirbulat Date: Wed, 17 Dec 2025 19:35:04 +0300 Subject: [PATCH 4/6] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B8=D0=BB=20=D0=BA?= =?UTF-8?q?=D0=BB=D0=B8=D0=BA,=20=D0=BD=D0=B0=D0=BF=D0=B8=D1=81=D0=B0?= =?UTF-8?q?=D0=BB=20=D0=B2=D0=B5=D0=B7=D0=B4=D0=B5=20=D0=BA=D0=BE=D0=BC?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=80=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bhabibullin/clickhouse/clickhouse_pet.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/bhabibullin/clickhouse/clickhouse_pet.sql b/bhabibullin/clickhouse/clickhouse_pet.sql index 68bd351..ab38da1 100644 --- a/bhabibullin/clickhouse/clickhouse_pet.sql +++ b/bhabibullin/clickhouse/clickhouse_pet.sql @@ -382,3 +382,4 @@ ORDER BY registration_date DESC; С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения */ + \ No newline at end of file From e7f7bb8dc6d13d0ce4cb624d0ec0268b9ef6745f Mon Sep 17 00:00:00 2001 From: Batirbulat Date: Wed, 24 Dec 2025 21:07:56 +0300 Subject: [PATCH 5/6] =?UTF-8?q?=D0=9A=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=B0=D1=80=D0=B8=D0=B8=20=D0=BD=D0=B0=D0=BF=D0=B8=D1=81=D0=B0?= =?UTF-8?q?=D0=BD=D1=8B,=20=D1=82=D0=B0=D0=B1=D0=BB=D0=B8=D1=86=D1=8B=20?= =?UTF-8?q?=D0=BF=D1=80=D0=B8=D0=B2=D0=B5=D0=B4=D0=B5=D0=BD=D1=8B=20=D0=B2?= =?UTF-8?q?=20=D0=BF=D0=BE=D1=80=D1=8F=D0=B4=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bhabibullin/clickhouse/clickhouse_pet.sql | 52 +++++++++++++++++------ 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/bhabibullin/clickhouse/clickhouse_pet.sql b/bhabibullin/clickhouse/clickhouse_pet.sql index ab38da1..94ad016 100644 --- a/bhabibullin/clickhouse/clickhouse_pet.sql +++ b/bhabibullin/clickhouse/clickhouse_pet.sql @@ -87,6 +87,7 @@ ORDER BY (lesson_id, course_id); */ CREATE TABLE bhabibullin.lesson_popularity_summary +ON CLUSTER clickhouse_cluster ( lesson_id UInt32, lesson_title String, @@ -97,10 +98,14 @@ CREATE TABLE bhabibullin.lesson_popularity_summary first_view DateTime64(3, 'UTC'), last_view DateTime64(3, 'UTC') ) -ENGINE = MergeTree +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary', + '{replica}' +) ORDER BY (lesson_id, course_id); CREATE MATERIALIZED VIEW mv_lesson_popularity_summary +ON CLUSTER clickhouse_cluster TO bhabibullin.lesson_popularity_summary AS SELECT @@ -140,8 +145,9 @@ GROUP BY flv.course_id, c.title; -select * from bhabibullin.mv_lesson_popularity_summary mlps ; -select * from bhabibullin.lesson_popularity_summary lps ; +select * from bhabibullin.mv_lesson_popularity_summary ; +select * from bhabibullin.lesson_popularity_summary ; +show tables; /* ### ✅ **Задание 3: Создай витрину `inactive_users_summary`** @@ -161,7 +167,13 @@ select * from bhabibullin.lesson_popularity_summary lps ; Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости. */ -CREATE TABLE bhabibullin.inactive_users_summary( +DROP TABLE bhabibullin.inactive_users_summary; +DROP VIEW bhabibullin.mv_inactive_users_summary ; +show tables; + +CREATE TABLE bhabibullin.inactive_users_summary +ON CLUSTER clickhouse_cluster +( user_id UInt32, name String, email String, @@ -169,10 +181,14 @@ CREATE TABLE bhabibullin.inactive_users_summary( enrollment_count UInt32, registration_date Date ) -ENGINE = MergeTree() +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/inactive_users_summary', + '{replica}' +) ORDER BY user_id; CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary +ON CLUSTER clickhouse_cluster to bhabibullin.inactive_users_summary AS SELECT @@ -204,6 +220,9 @@ WHERE (cdu.user_id, 1) NOT IN ( GROUP BY user_id ); +select * from bhabibullin.inactive_users_summary ius ; +select * from bhabibullin.mv_inactive_users_summary mius ; + /* ### ✅ **Задание 4: Создай витрину `course_completion_rate`** @@ -218,8 +237,8 @@ WHERE (cdu.user_id, 1) NOT IN ( - Коэффициент завершения (`lessons_viewed / lessons_in_course`) */ - CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate +ON CLUSTER clickhouse_cluster ( user_id UInt32, user_name String, @@ -229,11 +248,15 @@ CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate lessons_viewed UInt32, completion_rate Float64 ) -ENGINE = MergeTree() +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/course_completion_rate', + '{replica}' +) ORDER BY (user_id, course_id); CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate +ON CLUSTER clickhouse_cluster TO bhabibullin.course_completion_rate AS WITH lessons_in_course AS ( @@ -312,6 +335,9 @@ JOIN core_dim_course c ON e.course_id = c.course_id JOIN lessons_in_course AS lic ON c.course_id = lic.course_id LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id; +select * from bhabibullin.course_completion_rate; +select * from bhabibullin.mv_course_completion_rate; + /* # 📊 Этап 3. Анализ и запросы @@ -333,9 +359,9 @@ LIMIT 5; SELECT name, email, - registered_courses_count + enrollment_count FROM inactive_users_summary -WHERE registered_courses_count > 0 +WHERE enrollment_count > 0 ORDER BY registration_date DESC; --Курсы с самым высоким процентом завершения @@ -350,10 +376,9 @@ LIMIT 5; -- топ 5 курсов по количеству уникальных пользователей SELECT course_title, - uniqExact(user_id) AS active_users + unique_users FROM lesson_popularity_summary -GROUP BY course_title -ORDER BY active_users DESC +ORDER BY unique_users DESC LIMIT 5; --новые пользователи за прошлый месяц @@ -368,7 +393,7 @@ ORDER BY registration_date DESC; /* ### ✅ **Задание 6: Сравни данные с Greenplum** - + 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum 2. Сравни с ClickHouse-результатами: - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX` @@ -382,4 +407,3 @@ ORDER BY registration_date DESC; С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения */ - \ No newline at end of file From 3297ddb2b5b478a070475f931850b8e4d6c315fe Mon Sep 17 00:00:00 2001 From: Batirbulat Date: Wed, 31 Dec 2025 17:51:04 +0300 Subject: [PATCH 6/6] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=D1=81=D1=8F=20=D0=BF=D0=BE=20=D0=B2=D1=81=D0=B5=D0=BC=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=BC=D0=B5=D1=87=D0=B0=D0=BD=D0=B8=D1=8F=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bhabibullin/clickhouse/clickhouse_pet.sql | 50 ++++++++++++--------- bhabibullin/greenplum/pet_project_gp.sql | 17 +++---- bhabibullin/postgre/postrge_pet_project.sql | 12 +++-- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/bhabibullin/clickhouse/clickhouse_pet.sql b/bhabibullin/clickhouse/clickhouse_pet.sql index 94ad016..404abdc 100644 --- a/bhabibullin/clickhouse/clickhouse_pet.sql +++ b/bhabibullin/clickhouse/clickhouse_pet.sql @@ -86,6 +86,8 @@ ORDER BY (lesson_id, course_id); - Дата первого и последнего просмотра */ +-- Основная таблица с AggregatingMergeTree + CREATE TABLE bhabibullin.lesson_popularity_summary ON CLUSTER clickhouse_cluster ( @@ -93,18 +95,19 @@ ON CLUSTER clickhouse_cluster lesson_title String, course_id UInt32, course_title String, - total_views UInt32, - unique_users UInt32, - first_view DateTime64(3, 'UTC'), - last_view DateTime64(3, 'UTC') + total_views AggregateFunction(sum, UInt64), + unique_users AggregateFunction(uniq, UInt32), + first_view AggregateFunction(min, DateTime64(3, 'UTC')), + last_view AggregateFunction(max, DateTime64(3, 'UTC')) ) -ENGINE = ReplicatedMergeTree( - '/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary', - '{replica}' +ENGINE = ReplicatedAggregatingMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary', + '{replica}' ) -ORDER BY (lesson_id, course_id); +ORDER BY (lesson_id, course_id) +SETTINGS index_granularity = 8192; -CREATE MATERIALIZED VIEW mv_lesson_popularity_summary +CREATE MATERIALIZED VIEW bhabibullin.mv_lesson_popularity_summary ON CLUSTER clickhouse_cluster TO bhabibullin.lesson_popularity_summary AS @@ -113,13 +116,13 @@ SELECT l.title AS lesson_title, flv.course_id AS course_id, c.title AS course_title, - count() AS total_views, - uniqExact(flv.user_id) AS unique_users, - min(flv.viewed_at) AS first_view, - max(flv.viewed_at) AS last_view + sumState(1) AS total_views, + uniqState(flv.user_id) AS unique_users, + minState(flv.viewed_at) AS first_view, + maxState(flv.viewed_at) AS last_view FROM core_fact_lesson_views AS flv -JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id -JOIN core_dim_course AS c ON flv.course_id = c.course_id +JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN core_dim_course AS c ON flv.course_id = c.course_id GROUP BY flv.lesson_id, l.title, @@ -189,21 +192,24 @@ ORDER BY user_id; CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary ON CLUSTER clickhouse_cluster -to bhabibullin.inactive_users_summary +TO bhabibullin.inactive_users_summary AS SELECT cdu.user_id, cdu.name, cdu.email, cdu.age, - 0 as enrollment_count, + countDistinct(flv.course_id) as enrollment_count, cdu.registration_date FROM core_dim_user cdu -WHERE (cdu.user_id, 1) NOT IN ( - SELECT user_id, 1 - FROM core_enrollments - GROUP BY user_id -); +LEFT ANTI JOIN core_fact_lesson_views flv + ON cdu.user_id = flv.user_id +GROUP BY + cdu.user_id, + cdu.name, + cdu.email, + cdu.age, + cdu.registration_date; INSERT INTO bhabibullin.inactive_users_summary SELECT diff --git a/bhabibullin/greenplum/pet_project_gp.sql b/bhabibullin/greenplum/pet_project_gp.sql index d4cacd5..062f11b 100644 --- a/bhabibullin/greenplum/pet_project_gp.sql +++ b/bhabibullin/greenplum/pet_project_gp.sql @@ -119,16 +119,14 @@ SELECT dl.title AS lesson_title, dc.course_id, dc.title AS course_title, - COUNT(flv.lesson_id) AS total_views, - COUNT(DISTINCT du.user_id) AS unique_users, + COUNT(*) AS total_views, + COUNT(DISTINCT flv.user_id) AS unique_users, MIN(flv.viewed_at) AS first_view, MAX(flv.viewed_at) AS last_view FROM core.dim_lesson dl JOIN core.dim_course dc ON dc.course_id = dl.course_id JOIN core.fact_lesson_views flv ON flv.lesson_id = dl.lesson_id -JOIN core.dim_user du ON du.user_id = flv.user_id -GROUP BY dl.lesson_id, dl.title, dc.course_id, dc.title -ORDER BY dl.lesson_id; +GROUP BY dl.lesson_id, dl.title, dc.course_id, dc.title; /* @@ -155,12 +153,15 @@ SELECT du.name AS user_name, du.email, du.age, - COUNT(e.course_id) AS enrollments_count, + COUNT(DISTINCT e.course_id) AS enrollments_count, du.registration_date FROM core.dim_user du LEFT JOIN core.enrollments e ON e.user_id = du.user_id -LEFT JOIN core.fact_lesson_views flv ON flv.user_id = du.user_id -WHERE flv.user_id IS NULL +WHERE NOT EXISTS ( + SELECT 1 + FROM core.fact_lesson_views flv + WHERE flv.user_id = du.user_id +) GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date; /* diff --git a/bhabibullin/postgre/postrge_pet_project.sql b/bhabibullin/postgre/postrge_pet_project.sql index fa642d2..b4e0b86 100644 --- a/bhabibullin/postgre/postrge_pet_project.sql +++ b/bhabibullin/postgre/postrge_pet_project.sql @@ -90,12 +90,10 @@ select count(distinct lv.lesson_id) as viewed_lessons_count, max(lv.viewed_at) as last_view, min(lv.viewed_at) as first_view, - round( - count(distinct lv.lesson_id) * 100.0 / al.cnt_lesson - ,2) || '%' as engagegement_rate + round(count(distinct lv.lesson_id) * 100.0 / NULLIF(al.cnt_lesson, 0), 4) as engagement_rate from users u join enrollments e on u.id = e.user_id -join lesson_views lv on lv.user_id = u.id +left join lesson_views lv on lv.user_id = u.id join available_lessons al on al.user_id = u.id group by u.id, u.name, al.cnt_lesson ); @@ -136,7 +134,7 @@ from join lessons l on c.id = l.course_id join enrollments e on e.course_id = c.id join view_cnt vc on vc.course_id = c.id - join lesson_views lv on lv.lesson_id = l.id + left join lesson_views lv on lv.lesson_id = l.id group by c.title, vc.cnt, c.id order by course_id ); @@ -162,8 +160,8 @@ select count(distinct c.id) as total_courses, count(distinct l.id) as total_lessons, count(distinct lv.user_id) as users_with_views, - round(count(distinct l.id) * 1.0 / count(distinct c.id), 2) as avg_lessons_per_course, - count(lv.lesson_id) * 1.0 / count(distinct l.id) as avg_lessons_per_lesson + round(count(distinct l.id) * 1.0 / NULLIF(count(distinct c.id), 0), 2) as avg_lessons_per_course, + count(lv.lesson_id) * 1.0 / NULLIF(count(distinct l.id), 0) as avg_lessons_per_lesson from users u left join lesson_views lv on lv.user_id = u.id left join lessons l on l.id = lv.lesson_id