dmitry_medvedev/homework
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into anton

Browse Source
This commit is contained in:
aleksey bogatyrev
2026-01-17 16:47:17 +04:00
parent f98bc2632b 08e09f7515
commit c4550632f0
11 changed files with 1555 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amaksatov/postgresql/OS-329.sql
+215
View File
@@ -0,0 +1,215 @@
CREATE TABLE IF NOT EXISTS amaksatov.users (
id SERIAL PRIMARY KEY,
name VARCHAR(100),
age INT,
email VARCHAR(255),
registration_date DATE
);
CREATE TABLE IF NOT EXISTS amaksatov.courses (
id SERIAL PRIMARY KEY,
title VARCHAR(255),
category VARCHAR(100),
created_at DATE
);
CREATE TABLE IF NOT EXISTS amaksatov.lessons (
id SERIAL PRIMARY KEY,
course_id INT,
title VARCHAR(255),
duration_min INT,
FOREIGN KEY (course_id) REFERENCES amaksatov.courses(id)
);
CREATE TABLE IF NOT EXISTS amaksatov.enrollments (
id SERIAL PRIMARY KEY,
user_id INT,
course_id INT,
enrolled_at date,
FOREIGN KEY (user_id) REFERENCES amaksatov.users(id),
FOREIGN KEY (course_id) REFERENCES amaksatov.courses(id)
);
CREATE TABLE IF NOT EXISTS amaksatov.lesson_views (
id SERIAL PRIMARY KEY,
user_id INT,
lesson_id INT,
viewed_at TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES amaksatov.users(id),
FOREIGN KEY (lesson_id) REFERENCES amaksatov.lessons(id)
);
INSERT
INTO
amaksatov.users (
name,
age,
email,
registration_date
)
VALUES (
'Alice',
25,
'alice@mail.com',
'2023-01-10'
),
(
'Bob',
30,
'bob@gmail.com',
'2023-02-05'
),
(
'Charlie',
22,
'charlie@mail.com',
'2023-02-20'
),
(
'Diana',
28,
'diana@mail.com',
'2023-03-01'
),
(
'Ethan',
35,
'ethan@gmail.com',
'2023-03-10'
);
SELECT * FROM amaksatov.users;
INSERT
INTO
amaksatov.courses (
title,
category,
created_at
)
VALUES
(
'SQL для начинающих',
'data',
'2023-01-01'
),
(
'Python для анализа данных',
'programming',
'2023-01-15'
),
(
'BI с нуля',
'business',
'2023-03-01'
);
INSERT
INTO
amaksatov.lessons (
course_id,
title,
duration_min
)
VALUES
(
1,
'SELECT и FROM',
10
),
(
1,
'JOIN',
15
),
(
2,
'Pandas',
20
),
(
3,
'Основы BI',
12
),
(
3,
'Метрики и дашборды',
18
);
INSERT
INTO
amaksatov.enrollments (
user_id,
course_id,
enrolled_at
)
VALUES
(
1,
1,
'2023-01-15'
),
(
1,
2,
'2023-02-01'
),
(
2,
1,
'2023-01-20'
),
(
3,
2,
'2023-03-05'
),
(
4,
3,
'2023-04-01'
);
INSERT
INTO
amaksatov.lesson_views (
user_id,
lesson_id,
viewed_at
)
VALUES
(
1,
1,
'2023-01-16 10:00:00'
),
(
1,
2,
'2023-01-16 10:15:00'
),
(
2,
1,
'2023-01-21 09:00:00'
),
(
3,
3,
'2023-03-06 12:00:00'
),
(
4,
4,
'2023-04-02 10:00:00'
),
(
4,
5,
'2023-04-03 10:00:00'
);
amaksatov/postgresql/OS-330.sql
+68
View File
@@ -0,0 +1,68 @@
CREATE MATERIALIZED VIEW user_activity_summary AS
SELECT
u.id AS user_id, -- user_id
u.name, -- name
COUNT(DISTINCT e.course_id) AS enrollment_count, -- кол-во записей на курсы
COUNT(DISTINCT lv.lesson_id) AS viewed_lessons_count, -- кол-во просмотренных уроков
MAX(lv.viewed_at) AS last_viewed, -- последний просмотр
MIN(lv.viewed_at) AS first_viewed, -- первый просмотр
ROUND(COUNT(DISTINCT l.course_id) * 100.0 / COUNT(DISTINCT l.id), 2) || '%' AS engagement_rate -- вовлеченность(процент просмотренных/записанным)
FROM amaksatov.users u -- основная таблица
JOIN enrollments e ON e.user_id = u.id -- связь users с enrollments
LEFT JOIN lesson_views lv ON lv.user_id = e.user_id -- связь lessons с lesson_views
JOIN lessons l ON l.course_id = e.course_id -- связь уроков с курсами пользователя
GROUP BY u.id, u.name -- группировка по user_id и name
ORDER BY user_id; -- сортировка по user_id
CREATE MATERIALIZED VIEW course_summary AS
SELECT
c.id AS course_id,
c.title AS course_title,
COUNT(DISTINCT e.user_id) AS unique_users,
COUNT(DISTINCT l.id) AS lessons_count,
COUNT(DISTINCT lv.id) AS total_views,
ROUND(
CASE
WHEN COUNT(DISTINCT e.user_id) = 0 THEN NULL
ELSE COUNT(DISTINCT lv.id) * 1.0 / COUNT(DISTINCT e.user_id)
END, 2) AS avg_views_per_user,
MIN(lv.viewed_at) AS first_viewed_at,
MAX(lv.viewed_at) AS last_viewed_at
FROM courses c
LEFT JOIN enrollments e ON e.course_id = c.id
LEFT JOIN lessons l ON c.id = l.course_id
LEFT JOIN lesson_views lv ON lv.lesson_id = l.id
GROUP BY c.id;
CREATE MATERIALIZED VIEW platform_summary AS
WITH
user_stats AS (
SELECT COUNT(*) AS total_users FROM users
),
course_stats AS (
SELECT COUNT(*) AS total_courses FROM courses
),
lesson_stats AS (
SELECT COUNT(*) AS total_lessons FROM lessons
),
view_stats AS (
SELECT COUNT(*) AS total_views, COUNT(DISTINCT user_id) AS users_with_views
FROM lesson_views
)
SELECT
u.total_users,
c.total_courses,
l.total_lessons,
v.users_with_views,
ROUND(l.total_lessons::decimal / NULLIF(c.total_courses, 0), 2) AS avg_lessons_per_course,
ROUND(v.total_views::decimal / NULLIF(l.total_lessons, 0), 2) AS avg_views_per_lesson
FROM user_stats u, course_stats c, lesson_stats l, view_stats v;
bhabibullin/clickhouse/clickhouse_pet.sql
+415
View File
@@ -0,0 +1,415 @@
--ПЕТ ПРОЕКТ СОЗДАНИЕ ТАБЛИЦ
CREATE TABLE bhabibullin.core_enrollments
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
course_id UInt32,
enrolled_at DateTime
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_enrollments',
'{replica}'
)
ORDER BY (user_id, enrolled_at);
CREATE TABLE bhabibullin.core_dim_user
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
name String,
age UInt32,
email String,
registration_date Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_dim_user',
'{replica}'
)
ORDER BY (user_id, registration_date);
CREATE TABLE bhabibullin.core_fact_lesson_views
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
lesson_id UInt32,
course_id UInt32,
viewed_at DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_fact_lesson_views',
'{replica}'
)
ORDER BY user_id;
CREATE TABLE bhabibullin.core_dim_course
ON CLUSTER clickhouse_cluster
(
course_id UInt32,
title String,
category String
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_dim_course',
'{replica}'
)
ORDER BY course_id;
drop table bhabibullin.core_dim_course ;
CREATE TABLE bhabibullin.core_dim_lesson
ON CLUSTER clickhouse_cluster
(
lesson_id UInt32,
title String,
duration_min UInt32,
course_id UInt32
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/core_dim_lesson',
'{replica}'
)
ORDER BY (lesson_id, course_id);
-- Этап 2. Создание витринных таблиц
-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
/*
- Спроектируй таблицу, где каждая строка — один урок.
- Для каждого урока должны храниться:
- ID и название урока
- ID и название курса, к которому он относится
- Общее число просмотров
- Количество уникальных пользователей, посмотревших урок
- Дата первого и последнего просмотра
*/
-- Основная таблица с AggregatingMergeTree
CREATE TABLE bhabibullin.lesson_popularity_summary
ON CLUSTER clickhouse_cluster
(
lesson_id UInt32,
lesson_title String,
course_id UInt32,
course_title String,
total_views AggregateFunction(sum, UInt64),
unique_users AggregateFunction(uniq, UInt32),
first_view AggregateFunction(min, DateTime64(3, 'UTC')),
last_view AggregateFunction(max, DateTime64(3, 'UTC'))
)
ENGINE = ReplicatedAggregatingMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/lesson_popularity_summary',
'{replica}'
)
ORDER BY (lesson_id, course_id)
SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW bhabibullin.mv_lesson_popularity_summary
ON CLUSTER clickhouse_cluster
TO bhabibullin.lesson_popularity_summary
AS
SELECT
flv.lesson_id AS lesson_id,
l.title AS lesson_title,
flv.course_id AS course_id,
c.title AS course_title,
sumState(1) AS total_views,
uniqState(flv.user_id) AS unique_users,
minState(flv.viewed_at) AS first_view,
maxState(flv.viewed_at) AS last_view
FROM core_fact_lesson_views AS flv
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN core_dim_course AS c ON flv.course_id = c.course_id
GROUP BY
flv.lesson_id,
l.title,
flv.course_id,
c.title;
INSERT INTO bhabibullin.lesson_popularity_summary
SELECT
flv.lesson_id,
l.title AS lesson_title,
flv.course_id,
c.title AS course_title,
count() AS total_views,
uniqExact(flv.user_id) AS unique_users,
min(flv.viewed_at) AS first_view,
max(flv.viewed_at) AS last_view
FROM core_fact_lesson_views AS flv
JOIN core_dim_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN core_dim_course AS c ON flv.course_id = c.course_id
GROUP BY
flv.lesson_id,
l.title,
flv.course_id,
c.title;
select * from bhabibullin.mv_lesson_popularity_summary ;
select * from bhabibullin.lesson_popularity_summary ;
show tables;
/*
### ✅ **Задание 3: Создай витрину `inactive_users_summary`**
**Цель:** выявить пользователей, которые **зарегистрировались, но ничего не посмотрели**.
📌 **Что нужно сделать:**
- Подготовь таблицу, где каждая строка — один такой "неактивный" пользователь.
- Для каждого пользователя нужны:
- ID, имя, email, возраст
- Количество курсов, на которые он записался
- Дата регистрации
**Где пригодится:**
Эта витрина может использоваться для продуктовой аналитики, триггерных рассылок (например, напоминаний), анализа оттока, повышения вовлечённости.
*/
DROP TABLE bhabibullin.inactive_users_summary;
DROP VIEW bhabibullin.mv_inactive_users_summary ;
show tables;
CREATE TABLE bhabibullin.inactive_users_summary
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
name String,
email String,
age UInt32,
enrollment_count UInt32,
registration_date Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/inactive_users_summary',
'{replica}'
)
ORDER BY user_id;
CREATE MATERIALIZED VIEW bhabibullin.mv_inactive_users_summary
ON CLUSTER clickhouse_cluster
TO bhabibullin.inactive_users_summary
AS
SELECT
cdu.user_id,
cdu.name,
cdu.email,
cdu.age,
countDistinct(flv.course_id) as enrollment_count,
cdu.registration_date
FROM core_dim_user cdu
LEFT ANTI JOIN core_fact_lesson_views flv
ON cdu.user_id = flv.user_id
GROUP BY
cdu.user_id,
cdu.name,
cdu.email,
cdu.age,
cdu.registration_date;
INSERT INTO bhabibullin.inactive_users_summary
SELECT
cdu.user_id,
cdu.name,
cdu.email,
cdu.age,
0 as enrollment_count,
cdu.registration_date
FROM core_dim_user cdu
WHERE (cdu.user_id, 1) NOT IN (
SELECT user_id, 1
FROM core_enrollments
GROUP BY user_id
);
select * from bhabibullin.inactive_users_summary ius ;
select * from bhabibullin.mv_inactive_users_summary mius ;
/*
### ✅ **Задание 4: Создай витрину `course_completion_rate`**
**Цель:** проанализировать, насколько хорошо пользователи проходят курсы.
📌 **Что нужно сделать:**
- Сформируй таблицу, где строка — это пользователь + курс.
- Для каждой пары нужно рассчитать:
- Общее число уроков в курсе
- Сколько из них просмотрел конкретный пользователь
- Коэффициент завершения (`lessons_viewed / lessons_in_course`)
*/
CREATE TABLE IF NOT EXISTS bhabibullin.course_completion_rate
ON CLUSTER clickhouse_cluster
(
user_id UInt32,
user_name String,
course_id UInt32,
course_title String,
lessons_in_course UInt32,
lessons_viewed UInt32,
completion_rate Float64
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/bhabibullin/course_completion_rate',
'{replica}'
)
ORDER BY (user_id, course_id);
CREATE MATERIALIZED VIEW IF NOT EXISTS bhabibullin.mv_course_completion_rate
ON CLUSTER clickhouse_cluster
TO bhabibullin.course_completion_rate
AS
WITH lessons_in_course AS (
SELECT
course_id,
count(Distinct lesson_id) AS lessons_in_course
FROM core_dim_lesson
GROUP BY course_id
),
views_user_course AS (
SELECT
user_id,
course_id,
count(Distinct lesson_id) AS lessons_viewed
FROM core_fact_lesson_views
GROUP BY user_id, course_id
)
SELECT
u.user_id AS user_id,
u.name AS user_name,
c.course_id AS course_id,
c.title AS course_title,
lic.lessons_in_course AS lessons_in_course,
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
ROUND(
COALESCE(
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
/ NULLIF(lic.lessons_in_course * 1.0, 0),
0
),
2
) AS completion_rate
FROM core_dim_user u
JOIN core_enrollments e ON u.user_id = e.user_id
JOIN core_dim_course c ON e.course_id = c.course_id
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id
ORDER BY
u.user_id,
c.course_id;
INSERT INTO bhabibullin.course_completion_rate
WITH lessons_in_course AS (
SELECT
course_id,
count(Distinct lesson_id) AS lessons_in_course
FROM core_dim_lesson
GROUP BY course_id
),
views_user_course AS (
SELECT
user_id,
course_id,
count(Distinct lesson_id) AS lessons_viewed
FROM core_fact_lesson_views
GROUP BY user_id, course_id
)
SELECT
u.user_id AS user_id,
u.name AS user_name,
c.course_id AS course_id,
c.title AS course_title,
lic.lessons_in_course AS lessons_in_course,
coalesce(vuc.lessons_viewed, 0) AS lessons_viewed,
ROUND(
COALESCE(
NULLIF(coalesce(vuc.lessons_viewed, 0) * 1.0, 0)
/ NULLIF(lic.lessons_in_course * 1.0, 0),
0
),
2
) AS completion_rate
FROM core_dim_user u
JOIN core_enrollments e ON u.user_id = e.user_id
JOIN core_dim_course c ON e.course_id = c.course_id
JOIN lessons_in_course AS lic ON c.course_id = lic.course_id
LEFT JOIN views_user_course AS vuc ON vuc.user_id = u.user_id AND vuc.course_id = c.course_id;
select * from bhabibullin.course_completion_rate;
select * from bhabibullin.mv_course_completion_rate;
/*
# 📊 Этап 3. Анализ и запросы
### ✅ **Задание 5: Напиши SQL-запросы к витринам**
**Напиши свои вариации для практики.**
*/
--Топ-5 самых просматриваемых уроков
SELECT
lesson_title,
course_title,
total_views
FROM lesson_popularity_summary
ORDER BY total_views DESC
LIMIT 5;
--Неактивные пользователи, записавшиеся на курсы
SELECT
name,
email,
enrollment_count
FROM inactive_users_summary
WHERE enrollment_count > 0
ORDER BY registration_date DESC;
--Курсы с самым высоким процентом завершения
SELECT
course_title,
AVG(completion_rate) AS avg_completion
FROM course_completion_rate
GROUP BY course_title
ORDER BY avg_completion DESC
LIMIT 5;
-- топ 5 курсов по количеству уникальных пользователей
SELECT
course_title,
unique_users
FROM lesson_popularity_summary
ORDER BY unique_users DESC
LIMIT 5;
--новые пользователи за прошлый месяц
SELECT
user_id,
name,
email,
registration_date
FROM core_dim_user
WHERE registration_date >= today() - INTERVAL 30 DAY
ORDER BY registration_date DESC;
/*
### ✅ **Задание 6: Сравни данные с Greenplum**
1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
2. Сравни с ClickHouse-результатами:
- Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
- Проверяй `NULL`, `0`, округления
3. Напиши мини-отчёт: **"где совпадает, где нет, почему"
Совпадает ли SUM, AVG, MIN, MAX
Совпадает и в гринплам, и в кликхаус
Проверяй NULL, 0, округления
С этим даже траблы были при пет проекте, кликхаус тяжело обрабатывает нулл значения
*/
bhabibullin/greenplum/pet_project_gp.sql
+218
View File
@@ -0,0 +1,218 @@
CREATE SCHEMA raw;
CREATE SCHEMA core;
CREATE SCHEMA dm;
CREATE TABLE raw.users (
id INT,
name VARCHAR(255),
age INT,
email VARCHAR(255),
registration_date DATE
);
CREATE TABLE raw.courses (
id INT,
title VARCHAR(255),
category VARCHAR(255),
created_at DATE
);
CREATE TABLE raw.lessons (
id INT,
course_id INT,
title VARCHAR(255),
duration_min INT
);
CREATE TABLE raw.enrollments (
id INT,
user_id INT,
course_id INT,
enrolled_at DATE
);
CREATE TABLE raw.lesson_views (
id INT,
user_id INT,
lesson_id INT,
viewed_at TIMESTAMP
);
CREATE TABLE core.dim_user (
user_id INT PRIMARY KEY,
name TEXT,
age INT,
email TEXT,
registration_date DATE
);
CREATE TABLE core.dim_course (
course_id INT PRIMARY KEY,
title TEXT,
category TEXT
);
CREATE TABLE core.dim_lesson (
lesson_id INT PRIMARY KEY,
title TEXT,
duration_min INT,
course_id INT REFERENCES core.dim_course(course_id)
);
CREATE TABLE core.fact_lesson_views (
user_id INT REFERENCES core.dim_user(user_id),
lesson_id INT REFERENCES core.dim_lesson(lesson_id),
course_id INT REFERENCES core.dim_course(course_id),
viewed_at TIMESTAMP
);
CREATE TABLE core.enrollments (
user_id INT REFERENCES core.dim_user(user_id),
course_id INT REFERENCES core.dim_course(course_id),
enrolled_at TIMESTAMP
);
INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
SELECT DISTINCT
id::INT,
name::TEXT,
age::INT,
email::TEXT,
registration_date::DATE
FROM raw.users;
INSERT INTO core.dim_course (course_id, title, category)
SELECT DISTINCT
id::INT,
title::TEXT,
category::TEXT
FROM raw.courses;
INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
SELECT DISTINCT
id::INT,
title::TEXT,
duration_min::INT,
course_id::INT
FROM raw.lessons;
INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
SELECT DISTINCT
rw.user_id::INT,
rw.lesson_id::INT,
l.course_id::INT,
rw.viewed_at::TIMESTAMP
FROM raw.lesson_views rw
JOIN raw.lessons l ON rw.lesson_id = l.id;
INSERT INTO core.enrollments (user_id, course_id, enrolled_at)
SELECT DISTINCT
user_id::INT,
course_id::INT,
enrolled_at::DATE
FROM raw.enrollments;
CREATE MATERIALIZED VIEW dm.dm_lesson_popularity_summary AS
SELECT
dl.lesson_id,
dl.title AS lesson_title,
dc.course_id,
dc.title AS course_title,
COUNT(*) AS total_views,
COUNT(DISTINCT flv.user_id) AS unique_users,
MIN(flv.viewed_at) AS first_view,
MAX(flv.viewed_at) AS last_view
FROM core.dim_lesson dl
JOIN core.dim_course dc ON dc.course_id = dl.course_id
JOIN core.fact_lesson_views flv ON flv.lesson_id = dl.lesson_id
GROUP BY dl.lesson_id, dl.title, dc.course_id, dc.title;
/*
Цель:
Показать популярность каждого урока.
Поля витрины:
- lesson_id — идентификатор урока
- lesson_title — название урока
- course_id — идентификатор курса
- course_title — название курса
- total_views — общее число просмотров урока
- unique_users — количество уникальных пользователей
- first_view — дата первого просмотра
- last_view — дата последнего просмотра
*/
CREATE MATERIALIZED VIEW dm.dm_inactive_users_summary AS
SELECT
du.user_id,
du.name AS user_name,
du.email,
du.age,
COUNT(DISTINCT e.course_id) AS enrollments_count,
du.registration_date
FROM core.dim_user du
LEFT JOIN core.enrollments e ON e.user_id = du.user_id
WHERE NOT EXISTS (
SELECT 1
FROM core.fact_lesson_views flv
WHERE flv.user_id = du.user_id
)
GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date;
/*
Цель:
Определить невовлечённых пользователей и потенциальный отток.
Поля витрины:
- user_id — идентификатор пользователя
- user_name — имя
- email — почта
- age — возраст
- enrollments_count — количество курсов
- registration_date — дата регистрации
*/
CREATE MATERIALIZED VIEW dm.course_completion_rate AS
SELECT
du.user_id,
du.name AS user_name,
c.course_id,
c.title AS course_title,
COUNT(DISTINCT dl.lesson_id) AS lessons_in_course,
COUNT(DISTINCT flv.lesson_id) AS lessons_viewed,
ROUND(
COUNT(DISTINCT flv.lesson_id) * 1.0 /
NULLIF(COUNT(DISTINCT dl.lesson_id), 0),
2
) AS completion_rate
FROM core.dim_user du
JOIN core.enrollments e ON du.user_id = e.user_id
JOIN core.dim_course c ON e.course_id = c.course_id
JOIN core.dim_lesson dl ON c.course_id = dl.course_id
LEFT JOIN core.fact_lesson_views flv ON dl.lesson_id = flv.lesson_id
AND flv.user_id = du.user_id
GROUP BY du.user_id, du.name, c.course_id, c.title
ORDER BY du.user_id, c.course_id;
/*
Цель:
Анализировать, насколько пользователи проходят курсы до конца.
Поля витрины:
- user_id — идентификатор пользователя
- user_name — имя пользователя
- course_id — идентификатор курса
- course_title — название курса
- total_lessons — количество уроков в курсе
- lessons_viewed — число просмотренных уроков
- completion_rate — lessons_viewed / total_lessons
*/
bhabibullin/postgre/postrge_pet_project.sql
+182
View File
@@ -0,0 +1,182 @@
create table users(
id int primary key,
name varchar(255),
age int,
email varchar(255),
registration_date date
);
insert into users(id, name, age, email, registration_date) values
(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'),
(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'),
(3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'),
(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'),
(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10');
create table courses(
id int primary key,
title varchar(255),
category varchar(255),
created_at date
);
insert into courses(id, title, category, created_at) values
(1, 'SQL для начинающих', 'data', '2023-01-01'),
(2, 'Python для анализа данных', 'programming', '2023-01-15'),
(3, 'BI с нуля', 'business', '2023-03-01');
create table lessons(
id int primary key,
course_id int,
title varchar(255),
duration_min int,
foreign key (course_id) references courses(id)
);
insert into lessons(id, course_id, title, duration_min) values
(1, 1, 'SELECT и FROM', 10),
(2, 1, 'JOIN', 15),
(3, 2, 'Pandas', 20),
(4, 3, 'Основы BI', 12),
(5, 3, 'Метрики и дашборды', 18);
create table enrollments(
id int primary key,
user_id int,
course_id int,
enrolled_at date,
foreign key (user_id) references users(id),
foreign key (course_id) references courses(id)
);
insert into enrollments (id, user_id, course_id, enrolled_at) values
(1, 1, 1, '2023-01-15'),
(2, 1, 2, '2023-02-01'),
(3, 2, 1, '2023-01-20'),
(4, 3, 2, '2023-03-05'),
(5, 4, 3, '2023-04-01');
create table lesson_views(
id int primary key,
user_id int,
lesson_id int,
viewed_at timestamp,
foreign key (user_id) references users(id),
foreign key (lesson_id) references lessons(id)
);
insert into lesson_views(id, user_id, lesson_id, viewed_at) values
(1, 1, 1, '2023-01-16 10:00:00'),
(2, 1, 2, '2023-01-16 10:15:00'),
(3, 2, 1, '2023-01-21 09:00:00'),
(4, 3, 3, '2023-03-06 12:00:00'),
(5, 4, 4, '2023-04-02 10:00:00'),
(6, 4, 5, '2023-04-03 10:00:00');
create materialized view user_activity_summary as (
with available_lessons as (
select
e.user_id,
count(distinct l.id) as cnt_lesson
from lessons l
join enrollments e on e.course_id = l.course_id
group by e.user_id
)
select
u.id,
u.name,
count(distinct e.course_id) as enrrollment_count,
count(distinct lv.lesson_id) as viewed_lessons_count,
max(lv.viewed_at) as last_view,
min(lv.viewed_at) as first_view,
round(count(distinct lv.lesson_id) * 100.0 / NULLIF(al.cnt_lesson, 0), 4) as engagement_rate
from users u
join enrollments e on u.id = e.user_id
left join lesson_views lv on lv.user_id = u.id
join available_lessons al on al.user_id = u.id
group by u.id, u.name, al.cnt_lesson
);
/*
Витрина user_activity_summary
Цель - показать аналитику вовлеченности пользователей в обучение на платформе
user_id - идентификатор пользователя
name - имя пользователя
enrollment_count - количество записей на курсы
viewed_lessons_count - количество просмотренных уроков
last_view - последний просмотр
first_view - первый просмотр
engagement_rate - вовлеченность
*/
create materialized view course_summary as(
with view_cnt as (
select l.course_id,
count(lv.lesson_id) cnt
from lessons l
join lesson_views lv on l.id = lv.lesson_id
group by l.course_id
)
select
c.id as course_id,
c.title,
count(distinct e.user_id) unique_users,
count(distinct l.id) lessons_count,
vc.cnt total_views,
round(vc.cnt * 1.0 / count(distinct e.user_id), 2) as avg_views_per_user,
min(lv.viewed_at) as first_viewed_at,
max(lv.viewed_at) as last_viewed_at
from
courses c
join lessons l on c.id = l.course_id
join enrollments e on e.course_id = c.id
join view_cnt vc on vc.course_id = c.id
left join lesson_views lv on lv.lesson_id = l.id
group by c.title, vc.cnt, c.id
order by course_id
);
/*
Витрина course_summary
Цель - показать анализ эффективности по каждому курсу
course_id - идентификатор курса
course title - название курса
unique_users - кол-во уникальных пользователей в рамках одного курса
lessons_count - кол-во уроков
total_views - общее число просмотров уроков по курсу
avg_views_per_user - total_views / unique_users Среднее число просмотров на пользователя, записанного на курс
first_viewed_at - дата первого просмотра
last_viewed_at - дата последнего просмотра
*/
create materialized view platform_summary as (
select
count(distinct u.id) as total_users,
count(distinct c.id) as total_courses,
count(distinct l.id) as total_lessons,
count(distinct lv.user_id) as users_with_views,
round(count(distinct l.id) * 1.0 / NULLIF(count(distinct c.id), 0), 2) as avg_lessons_per_course,
count(lv.lesson_id) * 1.0 / NULLIF(count(distinct l.id), 0) as avg_lessons_per_lesson
from users u
left join lesson_views lv on lv.user_id = u.id
left join lessons l on l.id = lv.lesson_id
left join courses c on c.id = l.course_id
);
/*
Витрина platform_summary
Цель - показать общую отчетность по взаимодействию
total_users - всего пользователей
total_courses - всего курсов
total_lessons - всего уроков
users_with_views - сколько пользователей с минимум 1 просмотром
avg_lessons_per_course - среднее число уроков на курс
avg_views_per_lesson - среднее число просмотров на урок
*/
mdrozdov/postgresql/create_tables.sql
+70
View File
@@ -0,0 +1,70 @@
--drop table is exists users;
create table users(
id int primary key,
name varchar(50) not null,
age int not null,
email varchar(50) not null,
registration_date date not null default current_date
)
insert into users (id, name, age, email, registration_date) values
(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'),
(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'),
(3, 'Charlie',22, 'charlie@mail.com', '2023-02-20'),
(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'),
(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10')
create table courses(
id int primary key,
title text not null,
category varchar(30) not null,
created_at date not null default current_date
)
insert into courses (id, title, category, created_at) values
(1, 'SQL для начинающих', 'data', '2023-01-01'),
(2, 'Python для анализа данных', 'programming', '2023-01-15'),
(3, 'BI с нуля', 'business', '2023-03-01');
create table lessons(
id int primary key,
course_id int references courses(id),
title text not null,
duration_min int
)
insert into lessons (id, course_id, title, duration_min) values
(1, 1, 'SELECT и FROM', 10),
(2, 1, 'JOIN', 15),
(3, 2, 'Pandas', 20),
(4, 3, 'Основы BI', 12),
(5, 3, 'Метрики и дашборды', 18)
create table enrollments(
id int primary key,
user_id int references users(id),
course_id int references courses(id),
enrolled_at date not null default current_date
)
insert into enrollments (id, user_id, course_id, enrolled_at) values
(1, 1, 1, '2023-01-15'),
(2, 1, 2, '2023-02-01'),
(3, 2, 1, '2023-01-20'),
(4, 3, 2, '2023-03-05'),
(5, 4, 3, '2023-04-01')
create table lessons_views(
id int primary key,
user_id int references users(id),
lesson_id int references lessons(id),
viewed_at timestamp not null default now()
)
insert into lessons_views(id, user_id, lesson_id, viewed_at) values
(1, 1, 1, '2023-01-16 10:00:00'),
(2, 1, 2, '2023-01-16 10:15:00'),
(3, 2, 1, '2023-01-21 09:00:00'),
(4, 3, 3, '2023-03-06 12:00:00'),
(5, 4, 4, '2023-04-02 10:00:00'),
(6, 4, 5, '2023-04-03 10:00:00')
mdrozdov/postgresql/dm_course_summary.sql
+16
View File
@@ -0,0 +1,16 @@
create materialized view dm_course_summary as
select
e.course_id,
c.title,
count(distinct e.user_id) as unique_users,
count(distinct l.id) as lessons_count,
count(distinct lv.id) as total_views,
round(count(distinct lv.id)*1.0/nullif(count(distinct e.user_id), 0), 2) as avg_views_per_user,
min(lv.viewed_at) as first_viewed_at,
max(lv.viewed_at) as last_viewed_at
from enrollments e
join courses c on e.course_id = c.id
join lessons l on c.id = l.course_id
join lessons_views lv on lv.lesson_id = l.id
group by e.course_id, c.title
order by e.course_id
mdrozdov/postgresql/dm_platform_summary.sql
+8
View File
@@ -0,0 +1,8 @@
create materialized view dm_platform_summary as
select
(select count(id) from users) as total_users,
(select count(id) from courses) as total_courses,
(select count(id) from lessons) as total_lessons,
(select count(distinct user_id) from lessons_views) as users_with_views,
round((select count(id) from lessons)*1.0/nullif((select count(id) from courses), 0), 2) as avg_lessons_per_course,
round((select count(id) from lessons_views)*1.0/nullif((select count(id) from lessons), 0), 2) as avg_views_per_lesson
mdrozdov/postgresql/dm_user_activity_summary.sql
+84
View File
@@ -0,0 +1,84 @@
CREATE MATERIALIZED VIEW dm_user_activity_summary as
with enrollment_counts as(
select user_id,
count(course_id) as enrollment_count
from enrollments e
group by user_id
),views as(
select user_id,
count(distinct lesson_id) as viewed_lessons_count,
min(viewed_at) as first_view,
max(viewed_at) as last_view
from lessons_views
group by user_id
), total_lessons as(
select user_id,
count(*) as total_lessons
from enrollments e
join lessons using(course_id)
group by user_id
order by user_id
), engagment_rate as (
select tl.user_id, round(v.viewed_lessons_count*100.0/nullif(total_lessons), 2) as engagment_rate
from total_lessons tl
join views v using(user_id)
)
select u.id, u.name, enrollment_count, viewed_lessons_count, last_view, first_view, engagment_rate
from users u
join enrollment_counts ec on u.id = ec.user_id
join views v on u.id = v.user_id
join engagment_rate er on u.id = er.user_id
order by u.id
/*
--первая попытка сделать витрину, делал избыточно с проверкой каждого подзапроса, поэтому забыл убрать order by, а также больше практиковался в оконных функциях
CREATE MATERIALIZED VIEW user_activity_summary as
with enrollment_counts as(
select distinct user_id,
count(course_id) over (partition by user_id) as enrollment_count
from enrollments e
order by user_id
), viewed_lessons_count as (
select
user_id, count(distinct lesson_id) as viewed_lessons_count
from lessons_views lv
group by user_id
order by user_id
), first_views as(
select
user_id,
viewed_at,
row_number() over (partition by user_id order by viewed_at asc) as rnf
from lessons_views
), last_views as(
select
user_id,
viewed_at,
row_number() over (partition by user_id order by viewed_at desc) as rnl
from lessons_views
), result_views as(
select user_id,
fw.viewed_at as first_view,
lw.viewed_at as last_view
from first_views fw
join last_views lw using(user_id)
where rnf = 1 and rnl = 1
), total_lessons as(
select user_id,
count(*) as total_lessons
from enrollments e
join lessons using(course_id)
group by user_id
order by user_id
), engagment_rate as (
select tl.user_id, round(vlc.viewed_lessons_count*100.0/total_lessons, 2) as engagment_rate
from total_lessons tl
join viewed_lessons_count vlc using(user_id)
)
select u.id, u.name, enrollment_count, viewed_lessons_count, last_view, first_view, engagment_rate
from users u
join enrollment_counts ec on u.id = ec.user_id
join viewed_lessons_count vlc on u.id = vlc.user_id
join result_views rv on u.id = rv.user_id
join engagment_rate er on u.id = er.user_id
order by u.id
*/
ybirk/clickhouse/task_220_&_221.sql
+278
View File
@@ -0,0 +1,278 @@
CREATE TABLE ybirk.core_user ON CLUSTER clickhouse_cluster
(
user_id Int32,
name String,
age Int32,
email String,
registration_date Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_user',
'{replica}'
)
ORDER BY user_id;
CREATE TABLE ybirk.core_course ON CLUSTER clickhouse_cluster
(
course_id Int32,
title String,
category String
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_course',
'{replica}'
)
ORDER BY course_id;
CREATE TABLE ybirk.core_lesson ON CLUSTER clickhouse_cluster
(
lesson_id Int32,
title String,
duration_min Int32,
course_id Int32
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_lesson',
'{replica}'
)
ORDER BY lesson_id;
CREATE TABLE ybirk.core_fact_lesson_views ON CLUSTER clickhouse_cluster
(
user_id Int32,
lesson_id Int32,
course_id Int32,
viewed_at DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_fact_lesson_views',
'{replica}'
)
ORDER BY user_id;
CREATE TABLE ybirk.core_enrollments ON CLUSTER clickhouse_cluster
(
id Int32,
user_id Int32,
course_id Int32,
enrolled_at Date
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/ybirk/core_enrollments',
'{replica}'
)
ORDER BY user_id;
--------------------------------
CREATE TABLE ybirk.inactive_users_summary ON CLUSTER clickhouse_cluster
(
user_id UInt32,
name String,
email String,
age UInt8,
registered_courses_count UInt32,
registration_date DateTime64(3, 'UTC')
)
ENGINE = ReplicatedAggregatingMergeTree(
'/clickhouse/tables/{shard}/ybirk/inactive_users_summary',
'{replica}'
)
ORDER BY user_id;
CREATE TABLE ybirk.lesson_popularity_summary ON CLUSTER clickhouse_cluster
(
lesson_id UInt32,
lesson_title String,
course_id UInt32,
course_title String,
total_views UInt32,
unique_users UInt32,
first_view DateTime64(3, 'UTC'),
last_view DateTime64(3, 'UTC')
)
ENGINE = ReplicatedAggregatingMergeTree(
'/clickhouse/tables/{shard}/ybirk/lesson_popularity_summary',
'{replica}'
)
ORDER BY (lesson_id, course_id);
CREATE TABLE ybirk.course_completion_rate ON CLUSTER clickhouse_cluster
(
user_id UInt32,
user_name String,
course_id UInt32,
course_title String,
lessons_in_course UInt32,
lessons_viewed UInt32,
completion_rate Float32
)
ENGINE = ReplicatedAggregatingMergeTree(
'/clickhouse/tables/{shard}/ybirk/course_completion_rate',
'{replica}'
)
ORDER BY (user_id, course_id);
--------------------------------------
CREATE MATERIALIZED VIEW ybirk.mv_lesson_popularity_summary ON CLUSTER clickhouse_cluster
TO ybirk.lesson_popularity_summary
AS
SELECT
flv.lesson_id AS lesson_id,
l.title AS lesson_title,
flv.course_id AS course_id,
c.title AS course_title,
count() AS total_views,
uniqExact(flv.user_id) AS unique_users,
min(flv.viewed_at) AS first_view,
max(flv.viewed_at) AS last_view
FROM ybirk.core_fact_lesson_views AS flv
JOIN ybirk.core_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN ybirk.core_course AS c ON flv.course_id = c.course_id
GROUP BY
flv.lesson_id,
l.title,
flv.course_id,
c.title;
DROP VIEW IF EXISTS ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster;
CREATE MATERIALIZED VIEW ybirk.mv_inactive_users_summary ON CLUSTER clickhouse_cluster
TO ybirk.inactive_users_summary
AS
SELECT
u.user_id AS user_id,
u.name AS name,
u.email AS email,
u.age AS age,
countDistinct(e.course_id) AS registered_courses_count,
u.registration_date AS registration_date
FROM ybirk.core_user AS u
LEFT JOIN ybirk.core_enrollments AS e ON u.user_id = e.user_id
LEFT ANTI JOIN ybirk.core_fact_lesson_views AS flv ON u.user_id = flv.user_id
GROUP BY
u.user_id,
u.name,
u.email,
u.age,
u.registration_date;
CREATE MATERIALIZED VIEW ybirk.mv_course_completion_rate ON CLUSTER clickhouse_cluster
TO ybirk.course_completion_rate
AS
SELECT
u.user_id AS user_id,
u.name AS user_name,
c.course_id AS course_id,
c.title AS course_title,
countDistinct(l.lesson_id) AS lessons_in_course,
countDistinct(flv.lesson_id) AS lessons_viewed,
if(countDistinct(l.lesson_id) = 0, 0,
round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id), 2)
) AS completion_rate
FROM ybirk.core_enrollments AS e
JOIN ybirk.core_user AS u ON e.user_id = u.user_id
JOIN ybirk.core_course AS c ON e.course_id = c.course_id
JOIN ybirk.core_lesson AS l ON c.course_id = l.course_id
LEFT JOIN ybirk.core_fact_lesson_views AS flv
ON flv.user_id = e.user_id
AND flv.course_id = e.course_id
GROUP BY
u.user_id,
u.name,
c.course_id,
c.title;
----------------------------------------------
INSERT INTO ybirk.core_user (user_id, name, age, email, registration_date) VALUES
(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'),
(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'),
(3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'),
(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'),
(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10');
INSERT INTO ybirk.core_course (course_id, title, category) VALUES
(1, 'SQL для начинающих', 'data'),
(2, 'Python для анализа данных', 'programming'),
(3, 'BI с нуля', 'business');
INSERT INTO ybirk.core_lesson (lesson_id, course_id, title, duration_min) VALUES
(1, 1, 'SELECT и FROM', 10),
(2, 1, 'JOIN', 15),
(3, 2, 'Pandas', 20),
(4, 3, 'Основы BI', 12),
(5, 3, 'Метрики и дашборды', 18);
INSERT INTO ybirk.core_fact_lesson_views (user_id, lesson_id, course_id, viewed_at) VALUES
(1, 1, 1, '2023-01-16 10:00:00'),
(1, 2, 1, '2023-01-16 10:15:00'),
(2, 1, 1, '2023-01-21 09:00:00'),
(3, 3, 2, '2023-03-06 12:00:00'),
(4, 4, 3, '2023-04-02 10:00:00'),
(4, 5, 3, '2023-04-03 10:00:00');
INSERT INTO ybirk.core_enrollments (id, user_id, course_id, enrolled_at) VALUES
(1, 1, 1, '2023-01-15'),
(2, 1, 2, '2023-02-01'),
(3, 2, 1, '2023-01-20'),
(4, 3, 2, '2023-03-05'),
(5, 4, 3, '2023-04-01');
--------------------------------------------------
--Примеры
SELECT
lesson_title,
course_title,
total_views
FROM ybirk.mv_lesson_popularity_summary
ORDER BY total_views DESC
LIMIT 5;
SELECT
name,
email,
registered_courses_count
FROM ybirk.inactive_users_summary
WHERE registered_courses_count > 0
ORDER BY registration_date DESC;
SELECT
course_title,
AVG(completion_rate) AS avg_completion
FROM ybirk.course_completion_rate
GROUP BY course_title
ORDER BY avg_completion DESC
LIMIT 5;
-- ЗАпросы
-- самый популярный курс
SELECT
course_title,
SUM(total_views) as sum_views
FROM ybirk.mv_lesson_popularity_summary
GROUP BY course_title
ORDER BY sum_views DESC;
-- Кто застрял
SELECT
user_name,
course_title,
completion_rate
FROM ybirk.mv_course_completion_rate
WHERE completion_rate > 0 AND completion_rate < 1.0;
-- тест3
SELECT
t1.user_name,
t1.course_title,
t2.email
FROM ybirk.mv_course_completion_rate AS t1
JOIN ybirk.mv_inactive_users_summary AS t2
ON t1.user_id = t2.user_id
WHERE t1.completion_rate = 1.0;