diff --git a/amaksatov/postgresql/OS-329.sql b/amaksatov/postgresql/OS-329.sql new file mode 100644 index 0000000..3f0f080 --- /dev/null +++ b/amaksatov/postgresql/OS-329.sql @@ -0,0 +1,215 @@ +CREATE TABLE IF NOT EXISTS amaksatov.users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + age INT, + email VARCHAR(255), + registration_date DATE +); + +CREATE TABLE IF NOT EXISTS amaksatov.courses ( + id SERIAL PRIMARY KEY, + title VARCHAR(255), + category VARCHAR(100), + created_at DATE +); + +CREATE TABLE IF NOT EXISTS amaksatov.lessons ( + id SERIAL PRIMARY KEY, + course_id INT, + title VARCHAR(255), + duration_min INT, + FOREIGN KEY (course_id) REFERENCES amaksatov.courses(id) +); + +CREATE TABLE IF NOT EXISTS amaksatov.enrollments ( + id SERIAL PRIMARY KEY, + user_id INT, + course_id INT, + enrolled_at date, + FOREIGN KEY (user_id) REFERENCES amaksatov.users(id), + FOREIGN KEY (course_id) REFERENCES amaksatov.courses(id) +); + +CREATE TABLE IF NOT EXISTS amaksatov.lesson_views ( + id SERIAL PRIMARY KEY, + user_id INT, + lesson_id INT, + viewed_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES amaksatov.users(id), + FOREIGN KEY (lesson_id) REFERENCES amaksatov.lessons(id) +); + + +INSERT + INTO + amaksatov.users ( + name, + age, + email, + registration_date + ) +VALUES ( + 'Alice', + 25, + 'alice@mail.com', + '2023-01-10' +), +( + 'Bob', + 30, + 'bob@gmail.com', + '2023-02-05' +), +( + 'Charlie', + 22, + 'charlie@mail.com', + '2023-02-20' +), +( + 'Diana', + 28, + 'diana@mail.com', + '2023-03-01' +), +( + 'Ethan', + 35, + 'ethan@gmail.com', + '2023-03-10' +); + +SELECT * FROM amaksatov.users; + + +INSERT + INTO + amaksatov.courses ( + title, + category, + created_at + ) +VALUES +( + 'SQL для начинающих', + 'data', + '2023-01-01' +), +( + 'Python для анализа данных', + 'programming', + '2023-01-15' +), +( + 'BI с нуля', + 'business', + '2023-03-01' +); + +INSERT + INTO + amaksatov.lessons ( + course_id, + title, + duration_min + ) +VALUES +( + 1, + 'SELECT и FROM', + 10 +), +( + 1, + 'JOIN', + 15 +), +( + 2, + 'Pandas', + 20 +), +( + 3, + 'Основы BI', + 12 +), +( + 3, + 'Метрики и дашборды', + 18 +); + +INSERT + INTO + amaksatov.enrollments ( + user_id, + course_id, + enrolled_at + ) +VALUES +( + 1, + 1, + '2023-01-15' +), +( + 1, + 2, + '2023-02-01' +), +( + 2, + 1, + '2023-01-20' +), +( + 3, + 2, + '2023-03-05' +), +( + 4, + 3, + '2023-04-01' +); + +INSERT + INTO + amaksatov.lesson_views ( + user_id, + lesson_id, + viewed_at + ) +VALUES +( + 1, + 1, + '2023-01-16 10:00:00' +), +( + 1, + 2, + '2023-01-16 10:15:00' +), +( + 2, + 1, + '2023-01-21 09:00:00' +), +( + 3, + 3, + '2023-03-06 12:00:00' +), +( + 4, + 4, + '2023-04-02 10:00:00' +), +( + 4, + 5, + '2023-04-03 10:00:00' +); + diff --git a/amaksatov/postgresql/OS-330.sql b/amaksatov/postgresql/OS-330.sql new file mode 100644 index 0000000..9b78b52 --- /dev/null +++ b/amaksatov/postgresql/OS-330.sql @@ -0,0 +1,68 @@ + +CREATE MATERIALIZED VIEW user_activity_summary AS +SELECT + u.id AS user_id, -- user_id + u.name, -- name + COUNT(DISTINCT e.course_id) AS enrollment_count, -- кол-во записей на курсы + COUNT(DISTINCT lv.lesson_id) AS viewed_lessons_count, -- кол-во просмотренных уроков + MAX(lv.viewed_at) AS last_viewed, -- последний просмотр + MIN(lv.viewed_at) AS first_viewed, -- первый просмотр + ROUND(COUNT(DISTINCT l.course_id) * 100.0 / COUNT(DISTINCT l.id), 2) || '%' AS engagement_rate -- вовлеченность(процент просмотренных/записанным) +FROM amaksatov.users u -- основная таблица +JOIN enrollments e ON e.user_id = u.id -- связь users с enrollments +LEFT JOIN lesson_views lv ON lv.user_id = e.user_id -- связь lessons с lesson_views +JOIN lessons l ON l.course_id = e.course_id -- связь уроков с курсами пользователя +GROUP BY u.id, u.name -- группировка по user_id и name +ORDER BY user_id; -- сортировка по user_id + + +CREATE MATERIALIZED VIEW course_summary AS +SELECT + c.id AS course_id, + c.title AS course_title, + COUNT(DISTINCT e.user_id) AS unique_users, + COUNT(DISTINCT l.id) AS lessons_count, + COUNT(DISTINCT lv.id) AS total_views, + ROUND( + CASE + WHEN COUNT(DISTINCT e.user_id) = 0 THEN NULL + ELSE COUNT(DISTINCT lv.id) * 1.0 / COUNT(DISTINCT e.user_id) + END, 2) AS avg_views_per_user, + MIN(lv.viewed_at) AS first_viewed_at, + MAX(lv.viewed_at) AS last_viewed_at +FROM courses c +LEFT JOIN enrollments e ON e.course_id = c.id +LEFT JOIN lessons l ON c.id = l.course_id +LEFT JOIN lesson_views lv ON lv.lesson_id = l.id +GROUP BY c.id; + + + + + +CREATE MATERIALIZED VIEW platform_summary AS +WITH +user_stats AS ( + SELECT COUNT(*) AS total_users FROM users +), +course_stats AS ( + SELECT COUNT(*) AS total_courses FROM courses +), +lesson_stats AS ( + SELECT COUNT(*) AS total_lessons FROM lessons +), +view_stats AS ( + SELECT COUNT(*) AS total_views, COUNT(DISTINCT user_id) AS users_with_views + FROM lesson_views +) + +SELECT + u.total_users, + c.total_courses, + l.total_lessons, + v.users_with_views, + + ROUND(l.total_lessons::decimal / NULLIF(c.total_courses, 0), 2) AS avg_lessons_per_course, + ROUND(v.total_views::decimal / NULLIF(l.total_lessons, 0), 2) AS avg_views_per_lesson + +FROM user_stats u, course_stats c, lesson_stats l, view_stats v; diff --git a/bhabibullin/clickhouse/clickhouse_pet.sql b/bhabibullin/clickhouse/clickhouse_pet.sql new file mode 100644 index 0000000..404abdc --- /dev/null +++ b/bhabibullin/clickhouse/clickhouse_pet.sql @@ -0,0 +1,415 @@ +--ПЕТ ПРОЕКТ СОЗДАНИЕ ТАБЛИЦ + +CREATE TABLE bhabibullin.core_enrollments +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + course_id UInt32, + enrolled_at DateTime +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_enrollments', + '{replica}' +) +ORDER BY (user_id, enrolled_at); + + +CREATE TABLE bhabibullin.core_dim_user +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + name String, + age UInt32, + email String, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_dim_user', + '{replica}' +) +ORDER BY (user_id, registration_date); + + +CREATE TABLE bhabibullin.core_fact_lesson_views +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + lesson_id UInt32, + course_id UInt32, + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_fact_lesson_views', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE bhabibullin.core_dim_course +ON CLUSTER clickhouse_cluster +( + course_id UInt32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_dim_course', + '{replica}' +) +ORDER BY course_id; + +drop table bhabibullin.core_dim_course ; + +CREATE TABLE bhabibullin.core_dim_lesson +ON CLUSTER clickhouse_cluster +( + lesson_id UInt32, + title String, + duration_min UInt32, + course_id UInt32 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/core_dim_lesson', + '{replica}' +) +ORDER BY (lesson_id, course_id); + + +-- Этап 2. Создание витринных таблиц +-- Задание 2: Подготовь структуру витрины lesson_popularity_summary +/* +- Спроектируй таблицу, где каждая строка — один урок. +- Для каждого урока должны храниться: + - ID и название урока + - ID и название курса, к которому он относится + - Общее число просмотров + - Количество уникальных пользователей, посмотревших урок + - Дата первого и последнего просмотра + */ + +-- Основная таблица с AggregatingMergeTree + +CREATE TABLE bhabibullin.lesson_popularity_summary +ON CLUSTER clickhouse_cluster +( + lesson_id UInt32, + lesson_title String, + course_id UInt32, + course_title String, + total_views AggregateFunction(sum, UInt64), + unique_users AggregateFunction(uniq, UInt32), + first_view AggregateFunction(min, DateTime64(3, 'UTC')), + last_view AggregateFunction(max, DateTime64(3, 'UTC')) +) +ENGINE = ReplicatedAggregatingMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary', + '{replica}' +) +ORDER BY (lesson_id, course_id) +SETTINGS index_granularity = 8192; + +CREATE MATERIALIZED VIEW bhabibullin.mv_lesson_popularity_summary +ON CLUSTER clickhouse_cluster +TO bhabibullin.lesson_popularity_summary +AS +SELECT + flv.lesson_id AS lesson_id, + l.title AS lesson_title, + flv.course_id AS course_id, + c.title AS course_title, + sumState(1) AS total_views, + uniqState(flv.user_id) AS unique_users, + minState(flv.viewed_at) AS first_view, + maxState(flv.viewed_at) AS last_view +FROM core_fact_lesson_views AS flv +JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN core_dim_course AS c ON flv.course_id = c.course_id +GROUP BY + flv.lesson_id, + l.title, + flv.course_id, + c.title; + +INSERT INTO bhabibullin.lesson_popularity_summary +SELECT + flv.lesson_id, + l.title AS lesson_title, + flv.course_id, + c.title AS course_title, + count() AS total_views, + uniqExact(flv.user_id) AS unique_users, + min(flv.viewed_at) AS first_view, + max(flv.viewed_at) AS last_view +FROM core_fact_lesson_views AS flv +JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN core_dim_course AS c ON flv.course_id = c.course_id +GROUP BY + flv.lesson_id, + l.title, + flv.course_id, + c.title; + +select * from bhabibullin.mv_lesson_popularity_summary ; +select * from bhabibullin.lesson_popularity_summary ; +show tables; + +/* + ### ✅ **Задание 3: Создай витрину `inactive_users_summary`** + +**Цель:** выявить пользователей, которые **зарегистрировались, но ничего не посмотрели**. + +📌 **Что нужно сделать:** + +- Подготовь таблицу, где каждая строка — один такой "неактивный" пользователь. +- Для каждого пользователя нужны: + - ID, имя, email, возраст + - Количество курсов, на которые он записался + - Дата регистрации + +**Где пригодится:** + +Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости. + */ + +DROP TABLE bhabibullin.inactive_users_summary; +DROP VIEW bhabibullin.mv_inactive_users_summary ; +show tables; + +CREATE TABLE bhabibullin.inactive_users_summary +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + name String, + email String, + age UInt32, + enrollment_count UInt32, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/inactive_users_summary', + '{replica}' +) +ORDER BY user_id; + +CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary +ON CLUSTER clickhouse_cluster +TO bhabibullin.inactive_users_summary +AS +SELECT + cdu.user_id, + cdu.name, + cdu.email, + cdu.age, + countDistinct(flv.course_id) as enrollment_count, + cdu.registration_date +FROM core_dim_user cdu +LEFT ANTI JOIN core_fact_lesson_views flv + ON cdu.user_id = flv.user_id +GROUP BY + cdu.user_id, + cdu.name, + cdu.email, + cdu.age, + cdu.registration_date; + +INSERT INTO bhabibullin.inactive_users_summary +SELECT + cdu.user_id, + cdu.name, + cdu.email, + cdu.age, + 0 as enrollment_count, + cdu.registration_date +FROM core_dim_user cdu +WHERE (cdu.user_id, 1) NOT IN ( + SELECT user_id, 1 + FROM core_enrollments + GROUP BY user_id +); + +select * from bhabibullin.inactive_users_summary ius ; +select * from bhabibullin.mv_inactive_users_summary mius ; + +/* + ### ✅ **Задание 4: Создай витрину `course_completion_rate`** + +**Цель:** проанализировать, насколько хорошо пользователи проходят курсы. + +📌 **Что нужно сделать:** + +- Сформируй таблицу, где строка — это пользователь + курс. +- Для каждой пары нужно рассчитать: + - Общее число уроков в курсе + - Сколько из них просмотрел конкретный пользователь + - Коэффициент завершения (`lessons_viewed / lessons_in_course`) + */ + +CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate +ON CLUSTER clickhouse_cluster +( + user_id UInt32, + user_name String, + course_id UInt32, + course_title String, + lessons_in_course UInt32, + lessons_viewed UInt32, + completion_rate Float64 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/bhabibullin/course_completion_rate', + '{replica}' +) +ORDER BY (user_id, course_id); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate +ON CLUSTER clickhouse_cluster +TO bhabibullin.course_completion_rate +AS +WITH lessons_in_course AS ( + SELECT + course_id, + count(Distinct lesson_id) AS lessons_in_course + FROM core_dim_lesson + GROUP BY course_id +), +views_user_course AS ( + SELECT + user_id, + course_id, + count(Distinct lesson_id) AS lessons_viewed + FROM core_fact_lesson_views + GROUP BY user_id, course_id +) +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + lic.lessons_in_course AS lessons_in_course, + coalesce(vuc.lessons_viewed, 0) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0) + / NULLIF(lic.lessons_in_course * 1.0, 0), + 0 + ), + 2 + ) AS completion_rate +FROM core_dim_user u +JOIN core_enrollments e ON u.user_id = e.user_id +JOIN core_dim_course c ON e.course_id = c.course_id +JOIN lessons_in_course AS lic ON c.course_id = lic.course_id +LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id +ORDER BY + u.user_id, + c.course_id; + +INSERT INTO bhabibullin.course_completion_rate +WITH lessons_in_course AS ( + SELECT + course_id, + count(Distinct lesson_id) AS lessons_in_course + FROM core_dim_lesson + GROUP BY course_id +), +views_user_course AS ( + SELECT + user_id, + course_id, + count(Distinct lesson_id) AS lessons_viewed + FROM core_fact_lesson_views + GROUP BY user_id, course_id +) +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + lic.lessons_in_course AS lessons_in_course, + coalesce(vuc.lessons_viewed, 0) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0) + / NULLIF(lic.lessons_in_course * 1.0, 0), + 0 + ), + 2 + ) AS completion_rate +FROM core_dim_user u +JOIN core_enrollments e ON u.user_id = e.user_id +JOIN core_dim_course c ON e.course_id = c.course_id +JOIN lessons_in_course AS lic ON c.course_id = lic.course_id +LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id; + +select * from bhabibullin.course_completion_rate; +select * from bhabibullin.mv_course_completion_rate; + +/* + # 📊 Этап 3. Анализ и запросы + +### ✅ **Задание 5: Напиши SQL-запросы к витринам** + +**Напиши свои вариации для практики.** + */ + +--Топ-5 самых просматриваемых уроков +SELECT + lesson_title, + course_title, + total_views +FROM lesson_popularity_summary +ORDER BY total_views DESC +LIMIT 5; + +--Неактивные пользователи, записавшиеся на курсы +SELECT + name, + email, + enrollment_count +FROM inactive_users_summary +WHERE enrollment_count > 0 +ORDER BY registration_date DESC; + +--Курсы с самым высоким процентом завершения +SELECT + course_title, + AVG(completion_rate) AS avg_completion +FROM course_completion_rate +GROUP BY course_title +ORDER BY avg_completion DESC +LIMIT 5; + +-- топ 5 курсов по количеству уникальных пользователей +SELECT + course_title, + unique_users +FROM lesson_popularity_summary +ORDER BY unique_users DESC +LIMIT 5; + +--новые пользователи за прошлый месяц +SELECT + user_id, + name, + email, + registration_date +FROM core_dim_user +WHERE registration_date >= today() - INTERVAL 30 DAY +ORDER BY registration_date DESC; + +/* + ### ✅ **Задание 6: Сравни данные с Greenplum** + +1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum +2. Сравни с ClickHouse-результатами: + - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX` + - Проверяй `NULL`, `0`, округления +3. Напиши мини-отчёт: **"где совпадает, где нет, почему" + +Совпадает ли SUM, AVG, MIN, MAX +Совпадает и в гринплам, и в кликхаус + +Проверяй NULL, 0, округления +С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения +*/ + diff --git a/bhabibullin/greenplum/pet_project_gp.sql b/bhabibullin/greenplum/pet_project_gp.sql new file mode 100644 index 0000000..062f11b --- /dev/null +++ b/bhabibullin/greenplum/pet_project_gp.sql @@ -0,0 +1,218 @@ +CREATE SCHEMA raw; +CREATE SCHEMA core; +CREATE SCHEMA dm; + +CREATE TABLE raw.users ( + id INT, + name VARCHAR(255), + age INT, + email VARCHAR(255), + registration_date DATE +); + +CREATE TABLE raw.courses ( + id INT, + title VARCHAR(255), + category VARCHAR(255), + created_at DATE +); + +CREATE TABLE raw.lessons ( + id INT, + course_id INT, + title VARCHAR(255), + duration_min INT +); + +CREATE TABLE raw.enrollments ( + id INT, + user_id INT, + course_id INT, + enrolled_at DATE +); + +CREATE TABLE raw.lesson_views ( + id INT, + user_id INT, + lesson_id INT, + viewed_at TIMESTAMP +); + +CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE +); + +CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT +); + +CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) +); + +CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP +); + +CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP +); + +INSERT INTO core.dim_user (user_id, name, age, email, registration_date) +SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE +FROM raw.users; + +INSERT INTO core.dim_course (course_id, title, category) +SELECT DISTINCT + id::INT, + title::TEXT, + category::TEXT +FROM raw.courses; + +INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) +SELECT DISTINCT + id::INT, + title::TEXT, + duration_min::INT, + course_id::INT +FROM raw.lessons; + +INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) +SELECT DISTINCT + rw.user_id::INT, + rw.lesson_id::INT, + l.course_id::INT, + rw.viewed_at::TIMESTAMP +FROM raw.lesson_views rw +JOIN raw.lessons l ON rw.lesson_id = l.id; + + +INSERT INTO core.enrollments (user_id, course_id, enrolled_at) +SELECT DISTINCT + user_id::INT, + course_id::INT, + enrolled_at::DATE +FROM raw.enrollments; + +CREATE MATERIALIZED VIEW dm.dm_lesson_popularity_summary AS +SELECT + dl.lesson_id, + dl.title AS lesson_title, + dc.course_id, + dc.title AS course_title, + COUNT(*) AS total_views, + COUNT(DISTINCT flv.user_id) AS unique_users, + MIN(flv.viewed_at) AS first_view, + MAX(flv.viewed_at) AS last_view +FROM core.dim_lesson dl +JOIN core.dim_course dc ON dc.course_id = dl.course_id +JOIN core.fact_lesson_views flv ON flv.lesson_id = dl.lesson_id +GROUP BY dl.lesson_id, dl.title, dc.course_id, dc.title; + + +/* + + Цель: + Показать популярность каждого урока. + + Поля витрины: + - lesson_id — идентификатор урока + - lesson_title — название урока + - course_id — идентификатор курса + - course_title — название курса + - total_views — общее число просмотров урока + - unique_users — количество уникальных пользователей + - first_view — дата первого просмотра + - last_view — дата последнего просмотра + +*/ + + +CREATE MATERIALIZED VIEW dm.dm_inactive_users_summary AS +SELECT + du.user_id, + du.name AS user_name, + du.email, + du.age, + COUNT(DISTINCT e.course_id) AS enrollments_count, + du.registration_date +FROM core.dim_user du +LEFT JOIN core.enrollments e ON e.user_id = du.user_id +WHERE NOT EXISTS ( + SELECT 1 + FROM core.fact_lesson_views flv + WHERE flv.user_id = du.user_id +) +GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date; + +/* + +Цель: + Определить невовлечённых пользователей и потенциальный отток. + + Поля витрины: + - user_id — идентификатор пользователя + - user_name — имя + - email — почта + - age — возраст + - enrollments_count — количество курсов + - registration_date — дата регистрации + +*/ + +CREATE MATERIALIZED VIEW dm.course_completion_rate AS +SELECT + du.user_id, + du.name AS user_name, + c.course_id, + c.title AS course_title, + COUNT(DISTINCT dl.lesson_id) AS lessons_in_course, + COUNT(DISTINCT flv.lesson_id) AS lessons_viewed, + ROUND( + COUNT(DISTINCT flv.lesson_id) * 1.0 / + NULLIF(COUNT(DISTINCT dl.lesson_id), 0), + 2 + ) AS completion_rate +FROM core.dim_user du +JOIN core.enrollments e ON du.user_id = e.user_id +JOIN core.dim_course c ON e.course_id = c.course_id +JOIN core.dim_lesson dl ON c.course_id = dl.course_id +LEFT JOIN core.fact_lesson_views flv ON dl.lesson_id = flv.lesson_id + AND flv.user_id = du.user_id +GROUP BY du.user_id, du.name, c.course_id, c.title +ORDER BY du.user_id, c.course_id; + +/* + + Цель: + Анализировать, насколько пользователи проходят курсы до конца. + + Поля витрины: + - user_id — идентификатор пользователя + - user_name — имя пользователя + - course_id — идентификатор курса + - course_title — название курса + - total_lessons — количество уроков в курсе + - lessons_viewed — число просмотренных уроков + - completion_rate — lessons_viewed / total_lessons + +*/ \ No newline at end of file diff --git a/bhabibullin/postgre/postrge_pet_project.sql b/bhabibullin/postgre/postrge_pet_project.sql new file mode 100644 index 0000000..b4e0b86 --- /dev/null +++ b/bhabibullin/postgre/postrge_pet_project.sql @@ -0,0 +1,182 @@ +create table users( + id int primary key, + name varchar(255), + age int, + email varchar(255), + registration_date date +); + +insert into users(id, name, age, email, registration_date) values +(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'), +(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'), +(3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'), +(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'), +(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10'); + +create table courses( + id int primary key, + title varchar(255), + category varchar(255), + created_at date +); + +insert into courses(id, title, category, created_at) values +(1, 'SQL для начинающих', 'data', '2023-01-01'), +(2, 'Python для анализа данных', 'programming', '2023-01-15'), +(3, 'BI с нуля', 'business', '2023-03-01'); + +create table lessons( + id int primary key, + course_id int, + title varchar(255), + duration_min int, + foreign key (course_id) references courses(id) +); + +insert into lessons(id, course_id, title, duration_min) values +(1, 1, 'SELECT и FROM', 10), +(2, 1, 'JOIN', 15), +(3, 2, 'Pandas', 20), +(4, 3, 'Основы BI', 12), +(5, 3, 'Метрики и дашборды', 18); + +create table enrollments( + id int primary key, + user_id int, + course_id int, + enrolled_at date, + foreign key (user_id) references users(id), + foreign key (course_id) references courses(id) +); + +insert into enrollments (id, user_id, course_id, enrolled_at) values +(1, 1, 1, '2023-01-15'), +(2, 1, 2, '2023-02-01'), +(3, 2, 1, '2023-01-20'), +(4, 3, 2, '2023-03-05'), +(5, 4, 3, '2023-04-01'); + +create table lesson_views( + id int primary key, + user_id int, + lesson_id int, + viewed_at timestamp, + foreign key (user_id) references users(id), + foreign key (lesson_id) references lessons(id) +); + +insert into lesson_views(id, user_id, lesson_id, viewed_at) values +(1, 1, 1, '2023-01-16 10:00:00'), +(2, 1, 2, '2023-01-16 10:15:00'), +(3, 2, 1, '2023-01-21 09:00:00'), +(4, 3, 3, '2023-03-06 12:00:00'), +(5, 4, 4, '2023-04-02 10:00:00'), +(6, 4, 5, '2023-04-03 10:00:00'); + + +create materialized view user_activity_summary as ( +with available_lessons as ( + select + e.user_id, + count(distinct l.id) as cnt_lesson + from lessons l + join enrollments e on e.course_id = l.course_id + group by e.user_id +) +select + u.id, + u.name, + count(distinct e.course_id) as enrrollment_count, + count(distinct lv.lesson_id) as viewed_lessons_count, + max(lv.viewed_at) as last_view, + min(lv.viewed_at) as first_view, + round(count(distinct lv.lesson_id) * 100.0 / NULLIF(al.cnt_lesson, 0), 4) as engagement_rate +from users u +join enrollments e on u.id = e.user_id +left join lesson_views lv on lv.user_id = u.id +join available_lessons al on al.user_id = u.id +group by u.id, u.name, al.cnt_lesson +); + +/* + Витрина user_activity_summary + Цель - показать аналитику вовлеченности пользователей в обучение на платформе + + user_id - идентификатор пользователя + name - имя пользователя + enrollment_count - количество записей на курсы + viewed_lessons_count - количество просмотренных уроков + last_view - последний просмотр + first_view - первый просмотр + engagement_rate - вовлеченность + +*/ + +create materialized view course_summary as( +with view_cnt as ( + select l.course_id, + count(lv.lesson_id) cnt + from lessons l + join lesson_views lv on l.id = lv.lesson_id + group by l.course_id +) +select + c.id as course_id, + c.title, + count(distinct e.user_id) unique_users, + count(distinct l.id) lessons_count, + vc.cnt total_views, + round(vc.cnt * 1.0 / count(distinct e.user_id), 2) as avg_views_per_user, + min(lv.viewed_at) as first_viewed_at, + max(lv.viewed_at) as last_viewed_at +from + courses c + join lessons l on c.id = l.course_id + join enrollments e on e.course_id = c.id + join view_cnt vc on vc.course_id = c.id + left join lesson_views lv on lv.lesson_id = l.id +group by c.title, vc.cnt, c.id +order by course_id +); + +/* + Витрина course_summary + Цель - показать анализ эффективности по каждому курсу + + course_id - идентификатор курса + course title - название курса + unique_users - кол-во уникальных пользователей в рамках одного курса + lessons_count - кол-во уроков + total_views - общее число просмотров уроков по курсу + avg_views_per_user - total_views / unique_users Среднее число просмотров на пользователя, записанного на курс + first_viewed_at - дата первого просмотра + last_viewed_at - дата последнего просмотра +*/ + + +create materialized view platform_summary as ( +select + count(distinct u.id) as total_users, + count(distinct c.id) as total_courses, + count(distinct l.id) as total_lessons, + count(distinct lv.user_id) as users_with_views, + round(count(distinct l.id) * 1.0 / NULLIF(count(distinct c.id), 0), 2) as avg_lessons_per_course, + count(lv.lesson_id) * 1.0 / NULLIF(count(distinct l.id), 0) as avg_lessons_per_lesson +from users u +left join lesson_views lv on lv.user_id = u.id +left join lessons l on l.id = lv.lesson_id +left join courses c on c.id = l.course_id +); + +/* + Витрина platform_summary + Цель - показать общую отчетность по взаимодействию + + total_users - всего пользователей + total_courses - всего курсов + total_lessons - всего уроков + users_with_views - сколько пользователей с минимум 1 просмотром + avg_lessons_per_course - среднее число уроков на курс + avg_views_per_lesson - среднее число просмотров на урок + +*/ \ No newline at end of file diff --git a/kzhekshen/cl-house/queries/course_completion_rate.sql b/kzhekshen/cl-house/queries/course_completion_rate.sql index 7942256..232c79e 100644 --- a/kzhekshen/cl-house/queries/course_completion_rate.sql +++ b/kzhekshen/cl-house/queries/course_completion_rate.sql @@ -51,4 +51,4 @@ FROM pet_project.fact_lesson_views flv GROUP BY course_id ) cl ON cl.course_id = l.course_id GROUP BY flv.user_id, l.course_id -; +; \ No newline at end of file diff --git a/mdrozdov/postgresql/create_tables.sql b/mdrozdov/postgresql/create_tables.sql new file mode 100644 index 0000000..4b6f588 --- /dev/null +++ b/mdrozdov/postgresql/create_tables.sql @@ -0,0 +1,70 @@ +--drop table is exists users; +create table users( + id int primary key, + name varchar(50) not null, + age int not null, + email varchar(50) not null, + registration_date date not null default current_date +) + +insert into users (id, name, age, email, registration_date) values +(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'), +(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'), +(3, 'Charlie',22, 'charlie@mail.com', '2023-02-20'), +(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'), +(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10') + +create table courses( + id int primary key, + title text not null, + category varchar(30) not null, + created_at date not null default current_date +) + +insert into courses (id, title, category, created_at) values +(1, 'SQL для начинающих', 'data', '2023-01-01'), +(2, 'Python для анализа данных', 'programming', '2023-01-15'), +(3, 'BI с нуля', 'business', '2023-03-01'); + +create table lessons( + id int primary key, + course_id int references courses(id), + title text not null, + duration_min int +) + +insert into lessons (id, course_id, title, duration_min) values +(1, 1, 'SELECT и FROM', 10), +(2, 1, 'JOIN', 15), +(3, 2, 'Pandas', 20), +(4, 3, 'Основы BI', 12), +(5, 3, 'Метрики и дашборды', 18) + +create table enrollments( + id int primary key, + user_id int references users(id), + course_id int references courses(id), + enrolled_at date not null default current_date +) + +insert into enrollments (id, user_id, course_id, enrolled_at) values +(1, 1, 1, '2023-01-15'), +(2, 1, 2, '2023-02-01'), +(3, 2, 1, '2023-01-20'), +(4, 3, 2, '2023-03-05'), +(5, 4, 3, '2023-04-01') + +create table lessons_views( + id int primary key, + user_id int references users(id), + lesson_id int references lessons(id), + viewed_at timestamp not null default now() +) + +insert into lessons_views(id, user_id, lesson_id, viewed_at) values +(1, 1, 1, '2023-01-16 10:00:00'), +(2, 1, 2, '2023-01-16 10:15:00'), +(3, 2, 1, '2023-01-21 09:00:00'), +(4, 3, 3, '2023-03-06 12:00:00'), +(5, 4, 4, '2023-04-02 10:00:00'), +(6, 4, 5, '2023-04-03 10:00:00') \ No newline at end of file diff --git a/mdrozdov/postgresql/dm_course_summary.sql b/mdrozdov/postgresql/dm_course_summary.sql new file mode 100644 index 0000000..40b4b8d --- /dev/null +++ b/mdrozdov/postgresql/dm_course_summary.sql @@ -0,0 +1,16 @@ +create materialized view dm_course_summary as +select + e.course_id, + c.title, + count(distinct e.user_id) as unique_users, + count(distinct l.id) as lessons_count, + count(distinct lv.id) as total_views, + round(count(distinct lv.id)*1.0/nullif(count(distinct e.user_id), 0), 2) as avg_views_per_user, + min(lv.viewed_at) as first_viewed_at, + max(lv.viewed_at) as last_viewed_at +from enrollments e +join courses c on e.course_id = c.id +join lessons l on c.id = l.course_id +join lessons_views lv on lv.lesson_id = l.id +group by e.course_id, c.title +order by e.course_id diff --git a/mdrozdov/postgresql/dm_platform_summary.sql b/mdrozdov/postgresql/dm_platform_summary.sql new file mode 100644 index 0000000..146fde7 --- /dev/null +++ b/mdrozdov/postgresql/dm_platform_summary.sql @@ -0,0 +1,8 @@ +create materialized view dm_platform_summary as +select + (select count(id) from users) as total_users, + (select count(id) from courses) as total_courses, + (select count(id) from lessons) as total_lessons, + (select count(distinct user_id) from lessons_views) as users_with_views, + round((select count(id) from lessons)*1.0/nullif((select count(id) from courses), 0), 2) as avg_lessons_per_course, + round((select count(id) from lessons_views)*1.0/nullif((select count(id) from lessons), 0), 2) as avg_views_per_lesson \ No newline at end of file diff --git a/mdrozdov/postgresql/dm_user_activity_summary.sql b/mdrozdov/postgresql/dm_user_activity_summary.sql new file mode 100644 index 0000000..413d937 --- /dev/null +++ b/mdrozdov/postgresql/dm_user_activity_summary.sql @@ -0,0 +1,84 @@ +CREATE MATERIALIZED VIEW dm_user_activity_summary as +with enrollment_counts as( + select user_id, + count(course_id) as enrollment_count + from enrollments e + group by user_id +),views as( + select user_id, + count(distinct lesson_id) as viewed_lessons_count, + min(viewed_at) as first_view, + max(viewed_at) as last_view + from lessons_views + group by user_id +), total_lessons as( + select user_id, + count(*) as total_lessons + from enrollments e + join lessons using(course_id) + group by user_id + order by user_id +), engagment_rate as ( + select tl.user_id, round(v.viewed_lessons_count*100.0/nullif(total_lessons), 2) as engagment_rate + from total_lessons tl + join views v using(user_id) +) +select u.id, u.name, enrollment_count, viewed_lessons_count, last_view, first_view, engagment_rate +from users u +join enrollment_counts ec on u.id = ec.user_id +join views v on u.id = v.user_id +join engagment_rate er on u.id = er.user_id +order by u.id +/* +--первая попытка сделать витрину, делал избыточно с проверкой каждого подзапроса, поэтому забыл убрать order by, а также больше практиковался в оконных функциях +CREATE MATERIALIZED VIEW user_activity_summary as +with enrollment_counts as( + select distinct user_id, + count(course_id) over (partition by user_id) as enrollment_count + from enrollments e + order by user_id +), viewed_lessons_count as ( + select + user_id, count(distinct lesson_id) as viewed_lessons_count + from lessons_views lv + group by user_id + order by user_id +), first_views as( + select + user_id, + viewed_at, + row_number() over (partition by user_id order by viewed_at asc) as rnf + from lessons_views +), last_views as( + select + user_id, + viewed_at, + row_number() over (partition by user_id order by viewed_at desc) as rnl + from lessons_views +), result_views as( + select user_id, + fw.viewed_at as first_view, + lw.viewed_at as last_view + from first_views fw + join last_views lw using(user_id) + where rnf = 1 and rnl = 1 +), total_lessons as( + select user_id, + count(*) as total_lessons + from enrollments e + join lessons using(course_id) + group by user_id + order by user_id +), engagment_rate as ( + select tl.user_id, round(vlc.viewed_lessons_count*100.0/total_lessons, 2) as engagment_rate + from total_lessons tl + join viewed_lessons_count vlc using(user_id) +) +select u.id, u.name, enrollment_count, viewed_lessons_count, last_view, first_view, engagment_rate +from users u +join enrollment_counts ec on u.id = ec.user_id +join viewed_lessons_count vlc on u.id = vlc.user_id +join result_views rv on u.id = rv.user_id +join engagment_rate er on u.id = er.user_id +order by u.id +*/ \ No newline at end of file diff --git a/ybirk/clickhouse/task_220_&_221.sql b/ybirk/clickhouse/task_220_&_221.sql new file mode 100644 index 0000000..60480b9 --- /dev/null +++ b/ybirk/clickhouse/task_220_&_221.sql @@ -0,0 +1,278 @@ +CREATE TABLE ybirk.core_user ON CLUSTER clickhouse_cluster +( + user_id Int32, + name String, + age Int32, + email String, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ybirk/core_user', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE ybirk.core_course ON CLUSTER clickhouse_cluster +( + course_id Int32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ybirk/core_course', + '{replica}' +) +ORDER BY course_id; + +CREATE TABLE ybirk.core_lesson ON CLUSTER clickhouse_cluster +( + lesson_id Int32, + title String, + duration_min Int32, + course_id Int32 +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ybirk/core_lesson', + '{replica}' +) +ORDER BY lesson_id; + +CREATE TABLE ybirk.core_fact_lesson_views ON CLUSTER clickhouse_cluster +( + user_id Int32, + lesson_id Int32, + course_id Int32, + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ybirk/core_fact_lesson_views', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE ybirk.core_enrollments ON CLUSTER clickhouse_cluster +( + id Int32, + user_id Int32, + course_id Int32, + enrolled_at Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ybirk/core_enrollments', + '{replica}' +) +ORDER BY user_id; + +-------------------------------- + +CREATE TABLE ybirk.inactive_users_summary ON CLUSTER clickhouse_cluster +( + user_id UInt32, + name String, + email String, + age UInt8, + registered_courses_count UInt32, + registration_date DateTime64(3, 'UTC') +) +ENGINE = ReplicatedAggregatingMergeTree( + '/clickhouse/tables/{shard}/ybirk/inactive_users_summary', + '{replica}' +) +ORDER BY user_id; + +CREATE TABLE ybirk.lesson_popularity_summary ON CLUSTER clickhouse_cluster +( + lesson_id UInt32, + lesson_title String, + course_id UInt32, + course_title String, + total_views UInt32, + unique_users UInt32, + first_view DateTime64(3, 'UTC'), + last_view DateTime64(3, 'UTC') +) +ENGINE = ReplicatedAggregatingMergeTree( + '/clickhouse/tables/{shard}/ybirk/lesson_popularity_summary', + '{replica}' +) +ORDER BY (lesson_id, course_id); + +CREATE TABLE ybirk.course_completion_rate ON CLUSTER clickhouse_cluster +( + user_id UInt32, + user_name String, + course_id UInt32, + course_title String, + lessons_in_course UInt32, + lessons_viewed UInt32, + completion_rate Float32 +) +ENGINE = ReplicatedAggregatingMergeTree( + '/clickhouse/tables/{shard}/ybirk/course_completion_rate', + '{replica}' +) +ORDER BY (user_id, course_id); + +-------------------------------------- + +CREATE MATERIALIZED VIEW ybirk.mv_lesson_popularity_summary ON CLUSTER clickhouse_cluster +TO ybirk.lesson_popularity_summary +AS +SELECT + flv.lesson_id AS lesson_id, + l.title AS lesson_title, + flv.course_id AS course_id, + c.title AS course_title, + count() AS total_views, + uniqExact(flv.user_id) AS unique_users, + min(flv.viewed_at) AS first_view, + max(flv.viewed_at) AS last_view +FROM ybirk.core_fact_lesson_views AS flv +JOIN ybirk.core_lesson AS l ON flv.lesson_id = l.lesson_id +JOIN ybirk.core_course AS c ON flv.course_id = c.course_id +GROUP BY + flv.lesson_id, + l.title, + flv.course_id, + c.title; + +DROP VIEW IF EXISTS ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster; + +CREATE MATERIALIZED VIEW ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster +TO ybirk.inactive_users_summary +AS +SELECT + u.user_id AS user_id, + u.name AS name, + u.email AS email, + u.age AS age, + countDistinct(e.course_id) AS registered_courses_count, + u.registration_date AS registration_date +FROM ybirk.core_user AS u +LEFT JOIN ybirk.core_enrollments AS e ON u.user_id = e.user_id +LEFT ANTI JOIN ybirk.core_fact_lesson_views AS flv ON u.user_id = flv.user_id +GROUP BY + u.user_id, + u.name, + u.email, + u.age, + u.registration_date; + +CREATE MATERIALIZED VIEW ybirk.mv_course_completion_rate ON CLUSTER clickhouse_cluster +TO ybirk.course_completion_rate +AS +SELECT + u.user_id AS user_id, + u.name AS user_name, + c.course_id AS course_id, + c.title AS course_title, + countDistinct(l.lesson_id) AS lessons_in_course, + countDistinct(flv.lesson_id) AS lessons_viewed, + if(countDistinct(l.lesson_id) = 0, 0, + round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id), 2) + ) AS completion_rate +FROM ybirk.core_enrollments AS e +JOIN ybirk.core_user AS u ON e.user_id = u.user_id +JOIN ybirk.core_course AS c ON e.course_id = c.course_id +JOIN ybirk.core_lesson AS l ON c.course_id = l.course_id +LEFT JOIN ybirk.core_fact_lesson_views AS flv + ON flv.user_id = e.user_id + AND flv.course_id = e.course_id +GROUP BY + u.user_id, + u.name, + c.course_id, + c.title; + +---------------------------------------------- + +INSERT INTO ybirk.core_user (user_id, name, age, email, registration_date) VALUES +(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'), +(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'), +(3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'), +(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'), +(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10'); + +INSERT INTO ybirk.core_course (course_id, title, category) VALUES +(1, 'SQL для начинающих', 'data'), +(2, 'Python для анализа данных', 'programming'), +(3, 'BI с нуля', 'business'); + +INSERT INTO ybirk.core_lesson (lesson_id, course_id, title, duration_min) VALUES +(1, 1, 'SELECT и FROM', 10), +(2, 1, 'JOIN', 15), +(3, 2, 'Pandas', 20), +(4, 3, 'Основы BI', 12), +(5, 3, 'Метрики и дашборды', 18); + +INSERT INTO ybirk.core_fact_lesson_views (user_id, lesson_id, course_id, viewed_at) VALUES +(1, 1, 1, '2023-01-16 10:00:00'), +(1, 2, 1, '2023-01-16 10:15:00'), +(2, 1, 1, '2023-01-21 09:00:00'), +(3, 3, 2, '2023-03-06 12:00:00'), +(4, 4, 3, '2023-04-02 10:00:00'), +(4, 5, 3, '2023-04-03 10:00:00'); + +INSERT INTO ybirk.core_enrollments (id, user_id, course_id, enrolled_at) VALUES +(1, 1, 1, '2023-01-15'), +(2, 1, 2, '2023-02-01'), +(3, 2, 1, '2023-01-20'), +(4, 3, 2, '2023-03-05'), +(5, 4, 3, '2023-04-01'); + +-------------------------------------------------- + +--Примеры + +SELECT + lesson_title, + course_title, + total_views +FROM ybirk.mv_lesson_popularity_summary +ORDER BY total_views DESC +LIMIT 5; + +SELECT + name, + email, + registered_courses_count +FROM ybirk.inactive_users_summary +WHERE registered_courses_count > 0 +ORDER BY registration_date DESC; + +SELECT + course_title, + AVG(completion_rate) AS avg_completion +FROM ybirk.course_completion_rate +GROUP BY course_title +ORDER BY avg_completion DESC +LIMIT 5; + +-- ЗАпросы + +-- самый популярный курс +SELECT + course_title, + SUM(total_views) as sum_views +FROM ybirk.mv_lesson_popularity_summary +GROUP BY course_title +ORDER BY sum_views DESC; + + +-- Кто застрял +SELECT + user_name, + course_title, + completion_rate +FROM ybirk.mv_course_completion_rate +WHERE completion_rate > 0 AND completion_rate < 1.0; + +-- тест3 +SELECT + t1.user_name, + t1.course_title, + t2.email +FROM ybirk.mv_course_completion_rate AS t1 +JOIN ybirk.mv_inactive_users_summary AS t2 + ON t1.user_id = t2.user_id +WHERE t1.completion_rate = 1.0; \ No newline at end of file