dmitry_medvedev/homework
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Pet project is over

Browse Source
This commit is contained in:
Higor
2025-12-11 23:13:27 +03:00
parent 590ddcaa52
commit 9ed398b5cc
9 changed files with 265 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
itikhomirov/.DS_Store
Vendored
BIN
View File
Binary file not shown.
itikhomirov/clickhouse/Pet_project.sql
+265 -129
View File
@@ -1,69 +1,92 @@
CREATE database pet;
-- Этап 1 -- Этап 1
-- Задание 1 -- Задание 1
-- Перенести данные как в пет-проекте с Greenplum -- Перенести данные как в пет-проекте с Greenplum
-- Создаем таблицы, чтобы потом загрузить в них данные из таблиц GreenPlum CREATE TABLE itikhomirov.core_user
ON CLUSTER clickhouse_cluster
-- Таблица user из GP (
CREATE TABLE pet.core_user (
user_id Int32, user_id Int32,
name String, name String,
age Int32, age Int32,
email String, email String,
registration_date DateTime64(3, 'UTC') registration_date DateTime64(3, 'UTC')
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/core_user',
'{replica}'
)
ORDER BY user_id; ORDER BY user_id;
-- Таблица course из GP ALTER TABLE itikhomirov.core_user
CREATE TABLE pet.core_course ( MODIFY COLUMN registration_date DateTime64(3);
course_id Int32,
title String,
category String
)
ENGINE = MergeTree
ORDER BY course_id;
-- Таблица lesson из GP CREATE TABLE itikhomirov.core_lesson
CREATE TABLE pet.core_lesson ( ON CLUSTER clickhouse_cluster
(
lesson_id Int32, lesson_id Int32,
title String, title String,
duration_min Int32, duration_min Int32,
course_id Int32 course_id Int32
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/core_lesson',
'{replica}'
)
ORDER BY lesson_id; ORDER BY lesson_id;
-- Таблица fact_lesson_views из GP CREATE TABLE itikhomirov.core_fact_lesson_views
ON CLUSTER clickhouse_cluster
CREATE TABLE pet.core_fact_lesson_views ( (
user_id Int32, user_id Int32,
lesson_id Int32, lesson_id Int32,
course_id Int32, course_id Int32,
viewed_at DateTime64(3, 'UTC') viewed_at DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views',
'{replica}'
) )
ENGINE = MergeTree
ORDER BY user_id; ORDER BY user_id;
-- Таблица enrollments из GP ALTER TABLE itikhomirov.core_fact_lesson_views
MODIFY COLUMN viewed_at DateTime64(3);
CREATE TABLE pet.core_enrollments ( DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views
user_id Int32, ON CLUSTER clickhouse_cluster;
CREATE TABLE itikhomirov.core_enrollments
ON CLUSTER clickhouse_cluster
(
user_id Int32,
course_id Int32, course_id Int32,
enrolled_at DateTime64(3, 'UTC') enrolled_at DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы',
'{replica}'
) )
ENGINE = MergeTree
ORDER BY user_id; ORDER BY user_id;
CREATE TABLE itikhomirov.core_course
ON CLUSTER clickhouse_cluster
(
course_id Int32,
title String,
category String
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/core_course',
'{replica}'
)
ORDER BY course_id;
-- Этап 2. Создание витринных таблиц -- Этап 2. Создание витринных таблиц
-- Задание 2: Подготовь структуру витрины lesson_popularity_summary -- Задание 2: Подготовь структуру витрины lesson_popularity_summary
-- Создаем пустую таблицу lesson_popularity_summary(структуру витрины) -- Создаем пустую таблицу lesson_popularity_summary(структуру витрины)
CREATE TABLE pet.lesson_popularity_summary CREATE TABLE itikhomirov.lesson_popularity_summary
ON CLUSTER clickhouse_cluster
( (
lesson_id UInt32, lesson_id UInt32,
lesson_title String, lesson_title String,
@@ -71,16 +94,23 @@ CREATE TABLE pet.lesson_popularity_summary
course_title String, course_title String,
total_views UInt32, total_views UInt32,
unique_users UInt32, unique_users UInt32,
first_view DateTime64(3, 'UTC'), first_view DateTime64(3),
last_view DateTime64(3, 'UTC') last_view DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary',
'{replica}'
) )
ENGINE = MergeTree
ORDER BY (lesson_id, course_id); ORDER BY (lesson_id, course_id);
-- Создаем Materialized View -- Создаем Materialized View
CREATE MATERIALIZED VIEW pet.mv_lesson_popularity_summary DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary
TO pet.lesson_popularity_summary ON CLUSTER clickhouse_cluster;
CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary
ON CLUSTER clickhouse_cluster
TO itikhomirov.lesson_popularity_summary
AS AS
SELECT SELECT
flv.lesson_id AS lesson_id, flv.lesson_id AS lesson_id,
@@ -91,18 +121,18 @@ SELECT
uniqExact(flv.user_id) AS unique_users, uniqExact(flv.user_id) AS unique_users,
min(flv.viewed_at) AS first_view, min(flv.viewed_at) AS first_view,
max(flv.viewed_at) AS last_view max(flv.viewed_at) AS last_view
FROM pet.core_fact_lesson_views AS flv FROM itikhomirov.core_fact_lesson_views AS flv
JOIN pet.core_lesson AS l ON flv.lesson_id = l.lesson_id JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN pet.core_course AS c ON flv.course_id = c.course_id JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id
GROUP BY GROUP BY
flv.lesson_id, flv.lesson_id,
l.title, l.title,
flv.course_id, flv.course_id,
c.title; c.title;
-- Вставляем данные в таблицу pet.lesson_popularity_summary -- Вставляем данные в таблицу lesson_popularity_summary
INSERT INTO pet.lesson_popularity_summary INSERT INTO itikhomirov.lesson_popularity_summary
SELECT SELECT
flv.lesson_id, flv.lesson_id,
l.title AS lesson_title, l.title AS lesson_title,
@@ -112,9 +142,9 @@ SELECT
uniqExact(flv.user_id) AS unique_users, uniqExact(flv.user_id) AS unique_users,
min(flv.viewed_at) AS first_view, min(flv.viewed_at) AS first_view,
max(flv.viewed_at) AS last_view max(flv.viewed_at) AS last_view
FROM pet.core_fact_lesson_views AS flv FROM itikhomirov.core_fact_lesson_views AS flv
JOIN pet.core_lesson AS l ON flv.lesson_id = l.lesson_id JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id
JOIN pet.core_course AS c ON flv.course_id = c.course_id JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id
GROUP BY GROUP BY
flv.lesson_id, flv.lesson_id,
l.title, l.title,
@@ -125,75 +155,93 @@ GROUP BY
-- Создаем пустую таблицу inactive_users_summary(структуру витрины) -- Создаем пустую таблицу inactive_users_summary(структуру витрины)
CREATE TABLE pet.inactive_users_summary DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
ON CLUSTER clickhouse_cluster;
DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
ON CLUSTER clickhouse_cluster;
CREATE TABLE itikhomirov.inactive_users_summary
ON CLUSTER clickhouse_cluster
( (
user_id UInt32, user_id UInt32,
name String, name String,
email String, email String,
age UInt8, age UInt8,
registered_courses_count UInt32, registered_courses_count UInt32,
registration_date DateTime64(3, 'UTC') registration_date DateTime64(3)
)
ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary',
'{replica}'
) )
ENGINE = MergeTree
ORDER BY (user_id); ORDER BY (user_id);
-- Создаем Materialized View -- Создаем Materialized View
CREATE MATERIALIZED VIEW pet.mv_inactive_users_summary DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary
TO pet.inactive_users_summary ON CLUSTER clickhouse_cluster;
CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary
ON CLUSTER clickhouse_cluster
TO itikhomirov.inactive_users_summary
AS AS
WITH user_courses AS (
SELECT
e.user_id,
countDistinct(e.course_id) AS registered_courses_count
FROM itikhomirov.core_enrollments AS e
GROUP BY
e.user_id
)
SELECT
u.user_id AS user_id,
u.name AS name,
u.email AS email,
u.age AS age,
coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
u.registration_date AS registration_date
FROM itikhomirov.core_user AS u
LEFT JOIN user_courses AS uc
ON u.user_id = uc.user_id
WHERE u.user_id NOT IN (
SELECT DISTINCT user_id
FROM itikhomirov.core_fact_lesson_views
);
INSERT INTO itikhomirov.inactive_users_summary
WITH user_courses AS (
SELECT
e.user_id,
countDistinct(e.course_id) AS registered_courses_count
FROM itikhomirov.core_enrollments AS e
GROUP BY
e.user_id
)
-- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров
-- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views),
-- и к ним подмешивается уже готовый registered_courses_count.
SELECT SELECT
u.user_id AS user_id, u.user_id AS user_id,
u.name AS name, u.name AS name,
u.email AS email, u.email AS email,
u.age AS age, u.age AS age,
countDistinct(e.course_id) AS registered_courses_count, coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
u.registration_date AS registration_date u.registration_date AS registration_date
FROM pet.core_user AS u FROM itikhomirov.core_user AS u
LEFT JOIN pet.core_enrollments AS e ON u.user_id = e.user_id LEFT JOIN user_courses AS uc
ON u.user_id = uc.user_id
WHERE u.user_id NOT IN ( WHERE u.user_id NOT IN (
SELECT DISTINCT user_id SELECT DISTINCT user_id
FROM pet.core_fact_lesson_views FROM itikhomirov.core_fact_lesson_views
) )
GROUP BY ORDER BY u.user_id;
u.user_id,
u.name,
u.email,
u.age,
u.registration_date;
-- Вставляем данные в таблицу inactive_users_summary
INSERT INTO pet.inactive_users_summary
SELECT
u.user_id,
u.name,
u.email,
u.age,
countDistinct(e.course_id) AS registered_courses_count,
u.registration_date
FROM pet.core_user AS u
LEFT JOIN pet.core_enrollments AS e
ON u.user_id = e.user_id
WHERE u.user_id NOT IN (
SELECT DISTINCT user_id
FROM pet.core_fact_lesson_views
)
/*
* этот подзапрос WHERE... просто выбрасывает всех, кто хоть раз что‑то посмотрел,
* независимо от количества строк и особенностей JOIN’ов.
*/
GROUP BY
u.user_id,
u.name,
u.email,
u.age,
u.registration_date;
-- Задание 4: Создай витрину course_completion_rate -- Задание 4: Создай витрину course_completion_rate
CREATE TABLE pet.course_completion_rate CREATE TABLE itikhomirov.course_completion_rate
ON CLUSTER clickhouse_cluster
( (
user_id UInt32, user_id UInt32,
user_name String, user_name String,
@@ -203,65 +251,41 @@ CREATE TABLE pet.course_completion_rate
lessons_viewed UInt32, lessons_viewed UInt32,
completion_rate Float32 completion_rate Float32
) )
ENGINE = MergeTree ENGINE = ReplicatedMergeTree(
'/clickhouse/tables/{shard}/itikhomirov/course_completion_rate',
'{replica}'
)
ORDER BY (user_id, course_id); ORDER BY (user_id, course_id);
-- Создаем Materialized View -- Создаем Materialized View
CREATE MATERIALIZED VIEW pet.mv_course_completion_rate CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate
TO pet.course_completion_rate ON CLUSTER clickhouse_cluster
TO itikhomirov.course_completion_rate
AS AS
SELECT
u.user_id AS user_id,
u.name AS user_name,
c.course_id AS course_id,
c.title AS course_title,
countDistinct(l.lesson_id) AS lessons_in_course,
countDistinct(flv.lesson_id) AS lessons_viewed,
if(countDistinct(l.lesson_id) = 0,0,
round(countDistinct(flv.lesson_id) * 1.0 / countDistinct(l.lesson_id),2)
) AS completion_rate
FROM pet.core_enrollments AS e
JOIN pet.core_user AS u ON e.user_id = u.user_id
JOIN pet.core_course AS c ON e.course_id = c.course_id
JOIN pet.core_lesson AS l ON c.course_id = l.course_id
LEFT JOIN pet.core_fact_lesson_views AS flv
ON flv.user_id = e.user_id
AND flv.course_id = e.course_id
GROUP BY
u.user_id,
u.name,
c.course_id,
c.title;
-- Вставляем данные в таблицу pet.course_completion_rate
INSERT INTO pet.course_completion_rate
-- Создаем две временные таблицы:
-- 1) Сколько уроков в каждом курсе
WITH WITH
lessons_per_course AS ( lessons_per_course AS (
SELECT SELECT
course_id, course_id,
countDistinct(lesson_id) AS lessons_in_course countDistinct(lesson_id) AS lessons_in_course
FROM pet.core_lesson FROM itikhomirov.core_lesson
GROUP BY course_id GROUP BY course_id
), ),
-- 2) Сколько уроков каждого курса посмотрел пользователь
views_per_user_course AS ( views_per_user_course AS (
SELECT SELECT
user_id, user_id,
course_id, course_id,
countDistinct(lesson_id) AS lessons_viewed countDistinct(lesson_id) AS lessons_viewed
FROM pet.core_fact_lesson_views FROM itikhomirov.core_fact_lesson_views
GROUP BY user_id, course_id GROUP BY user_id, course_id
) )
-- Основной запрос
SELECT SELECT
u.user_id, u.user_id AS user_id,
u.name AS user_name, u.name AS user_name,
c.course_id, c.course_id AS course_id,
c.title AS course_title, c.title AS course_title,
lpc.lessons_in_course, lpc.lessons_in_course AS lessons_in_course,
coalesce(vpc.lessons_viewed, 0) AS lessons_viewed, coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
ROUND( ROUND(
COALESCE( COALESCE(
@@ -271,12 +295,124 @@ SELECT
), ),
2 2
) AS completion_rate ) AS completion_rate
FROM pet.core_user AS u FROM itikhomirov.core_user AS u
JOIN pet.core_enrollments AS e ON u.user_id = e.user_id JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id
JOIN pet.core_course AS c ON e.course_id = c.course_id JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id
ORDER BY ORDER BY
u.user_id, u.user_id,
c.course_id; c.course_id;
-- Вставляем данные в таблицу pet.course_completion_rate
INSERT INTO itikhomirov.course_completion_rate
-- Создаем две временные таблицы:
-- 1) Сколько уроков в каждом курсе
WITH
lessons_per_course AS (
SELECT
course_id,
countDistinct(lesson_id) AS lessons_in_course
FROM itikhomirov.core_lesson
GROUP BY course_id
),
-- 2) Сколько уроков каждого курса посмотрел пользователь
views_per_user_course AS (
SELECT
user_id,
course_id,
countDistinct(lesson_id) AS lessons_viewed
FROM itikhomirov.core_fact_lesson_views
GROUP BY user_id, course_id
)
-- Основной запрос
SELECT
u.user_id AS user_id,
u.name AS user_name,
c.course_id AS course_id,
c.title AS course_title,
lpc.lessons_in_course AS lessons_in_course,
coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
ROUND(
COALESCE(
NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
/ NULLIF(lpc.lessons_in_course * 1.0, 0),
0
),
2
) AS completion_rate
FROM itikhomirov.core_user AS u
JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id
JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id
ORDER BY
u.user_id,
c.course_id;
-- Этап 3. Анализ и запросы
-- Задание 5: Напиши SQL-запросы к витринам
-- Уроки с хотя бы одним просмотром
SELECT
lesson_title,
course_title,
total_views
FROM itikhomirov.lesson_popularity_summary
WHERE total_views > 0
ORDER BY lesson_title ASC;
-- Топ‑3 урока внутри одного курса
SELECT
lesson_title,
course_title,
total_views
FROM itikhomirov.lesson_popularity_summary
WHERE course_title = 'SQL для начинающих'
ORDER BY total_views DESC
LIMIT 3;
-- Пользователи, которые завершили курсы на 100%
SELECT
user_id,
user_name,
course_title,
completion_rate
FROM itikhomirov.course_completion_rate
WHERE completion_rate = 1
ORDER BY user_id;
-- Этап 4. Тестирование и валидация
-- Задание 6: Сравни данные с Greenplum**
-- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
-- 2. Сравни с ClickHouse-результатами:
-- - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
-- - Проверяй `NULL`, `0`, округления
-- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
SELECT
sum(total_views) AS sum_total_views,
avg(total_views) AS avg_total_views,
min(total_views) AS min_total_views,
max(total_views) AS max_total_views
FROM itikhomirov.lesson_popularity_summary
WHERE lesson_id IN (1, 3, 5);
SELECT
sum(total_views) AS sum_total_views,
avg(total_views) AS avg_total_views,
min(total_views) AS min_total_views,
max(total_views) AS max_total_views
FROM itikhomirov.lesson_popularity_summary
WHERE lesson_id = 999;
-- Отчёт в .doc файле
itikhomirov/clickhouse/Pet_project_Step 1.1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 292 KiB

itikhomirov/clickhouse/Pet_project_Step 1.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 114 KiB

itikhomirov/clickhouse/Pet_project_Step 2.2.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 210 KiB

After

Width:  |  Height:  |  Size: 359 KiB

itikhomirov/clickhouse/Pet_project_Step 2.3.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 41 KiB

After

Width:  |  Height:  |  Size: 215 KiB

itikhomirov/clickhouse/Pet_project_Step 2.4.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 87 KiB

After

Width:  |  Height:  |  Size: 129 KiB

itikhomirov/clickhouse/Pet_project_Step 3.5.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 234 KiB

itikhomirov/clickhouse/Step 4.6.pdf
BIN
View File
Binary file not shown.