Higor
419 lines
12 KiB
SQL
419 lines
12 KiB
SQL
-- Этап 1
|
||
-- Задание 1
|
||
-- Перенести данные как в пет-проекте с Greenplum
|
||
|
||
CREATE TABLE itikhomirov.core_user
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id Int32,
|
||
name String,
|
||
age Int32,
|
||
email String,
|
||
registration_date DateTime64(3, 'UTC')
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/core_user',
|
||
'{replica}'
|
||
)
|
||
ORDER BY user_id;
|
||
|
||
ALTER TABLE itikhomirov.core_user
|
||
MODIFY COLUMN registration_date DateTime64(3);
|
||
|
||
CREATE TABLE itikhomirov.core_lesson
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
lesson_id Int32,
|
||
title String,
|
||
duration_min Int32,
|
||
course_id Int32
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/core_lesson',
|
||
'{replica}'
|
||
)
|
||
ORDER BY lesson_id;
|
||
|
||
CREATE TABLE itikhomirov.core_fact_lesson_views
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id Int32,
|
||
lesson_id Int32,
|
||
course_id Int32,
|
||
viewed_at DateTime64(3)
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views',
|
||
'{replica}'
|
||
)
|
||
ORDER BY user_id;
|
||
|
||
ALTER TABLE itikhomirov.core_fact_lesson_views
|
||
MODIFY COLUMN viewed_at DateTime64(3);
|
||
|
||
DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views
|
||
ON CLUSTER clickhouse_cluster;
|
||
|
||
CREATE TABLE itikhomirov.core_enrollments
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id Int32,
|
||
course_id Int32,
|
||
enrolled_at DateTime64(3)
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы',
|
||
'{replica}'
|
||
)
|
||
ORDER BY user_id;
|
||
|
||
CREATE TABLE itikhomirov.core_course
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
course_id Int32,
|
||
title String,
|
||
category String
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/core_course',
|
||
'{replica}'
|
||
)
|
||
ORDER BY course_id;
|
||
|
||
-- Этап 2. Создание витринных таблиц
|
||
-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
|
||
|
||
-- Создаем пустую таблицу lesson_popularity_summary(структуру витрины)
|
||
|
||
CREATE TABLE itikhomirov.lesson_popularity_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
lesson_id UInt32,
|
||
lesson_title String,
|
||
course_id UInt32,
|
||
course_title String,
|
||
total_views UInt32,
|
||
unique_users UInt32,
|
||
first_view DateTime64(3),
|
||
last_view DateTime64(3)
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (lesson_id, course_id);
|
||
|
||
-- Создаем Materialized View
|
||
|
||
DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary
|
||
ON CLUSTER clickhouse_cluster;
|
||
|
||
CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
TO itikhomirov.lesson_popularity_summary
|
||
AS
|
||
SELECT
|
||
flv.lesson_id AS lesson_id,
|
||
l.title AS lesson_title,
|
||
flv.course_id AS course_id,
|
||
c.title AS course_title,
|
||
count() AS total_views,
|
||
uniqExact(flv.user_id) AS unique_users,
|
||
min(flv.viewed_at) AS first_view,
|
||
max(flv.viewed_at) AS last_view
|
||
FROM itikhomirov.core_fact_lesson_views AS flv
|
||
JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id
|
||
JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id
|
||
GROUP BY
|
||
flv.lesson_id,
|
||
l.title,
|
||
flv.course_id,
|
||
c.title;
|
||
|
||
-- Вставляем данные в таблицу lesson_popularity_summary
|
||
|
||
INSERT INTO itikhomirov.lesson_popularity_summary
|
||
SELECT
|
||
flv.lesson_id,
|
||
l.title AS lesson_title,
|
||
flv.course_id,
|
||
c.title AS course_title,
|
||
count() AS total_views,
|
||
uniqExact(flv.user_id) AS unique_users,
|
||
min(flv.viewed_at) AS first_view,
|
||
max(flv.viewed_at) AS last_view
|
||
FROM itikhomirov.core_fact_lesson_views AS flv
|
||
JOIN itikhomirov.core_lesson AS l ON flv.lesson_id = l.lesson_id
|
||
JOIN itikhomirov.core_course AS c ON flv.course_id = c.course_id
|
||
GROUP BY
|
||
flv.lesson_id,
|
||
l.title,
|
||
flv.course_id,
|
||
c.title;
|
||
|
||
-- Задание 3: Создай витрину inactive_users_summary
|
||
|
||
-- Создаем пустую таблицу inactive_users_summary(структуру витрины)
|
||
|
||
DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
|
||
ON CLUSTER clickhouse_cluster;
|
||
|
||
DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
|
||
ON CLUSTER clickhouse_cluster;
|
||
|
||
CREATE TABLE itikhomirov.inactive_users_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id UInt32,
|
||
name String,
|
||
email String,
|
||
age UInt8,
|
||
registered_courses_count UInt32,
|
||
registration_date DateTime64(3)
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (user_id);
|
||
|
||
-- Создаем Materialized View
|
||
|
||
DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary
|
||
ON CLUSTER clickhouse_cluster;
|
||
|
||
CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary
|
||
ON CLUSTER clickhouse_cluster
|
||
TO itikhomirov.inactive_users_summary
|
||
AS
|
||
WITH user_courses AS (
|
||
SELECT
|
||
e.user_id,
|
||
countDistinct(e.course_id) AS registered_courses_count
|
||
FROM itikhomirov.core_enrollments AS e
|
||
GROUP BY
|
||
e.user_id
|
||
)
|
||
SELECT
|
||
u.user_id AS user_id,
|
||
u.name AS name,
|
||
u.email AS email,
|
||
u.age AS age,
|
||
coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
|
||
u.registration_date AS registration_date
|
||
FROM itikhomirov.core_user AS u
|
||
LEFT JOIN user_courses AS uc
|
||
ON u.user_id = uc.user_id
|
||
WHERE u.user_id NOT IN (
|
||
SELECT DISTINCT user_id
|
||
FROM itikhomirov.core_fact_lesson_views
|
||
);
|
||
|
||
|
||
INSERT INTO itikhomirov.inactive_users_summary
|
||
WITH user_courses AS (
|
||
SELECT
|
||
e.user_id,
|
||
countDistinct(e.course_id) AS registered_courses_count
|
||
FROM itikhomirov.core_enrollments AS e
|
||
GROUP BY
|
||
e.user_id
|
||
)
|
||
-- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров
|
||
-- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views),
|
||
-- и к ним подмешивается уже готовый registered_courses_count.
|
||
SELECT
|
||
u.user_id AS user_id,
|
||
u.name AS name,
|
||
u.email AS email,
|
||
u.age AS age,
|
||
coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
|
||
u.registration_date AS registration_date
|
||
FROM itikhomirov.core_user AS u
|
||
LEFT JOIN user_courses AS uc
|
||
ON u.user_id = uc.user_id
|
||
WHERE u.user_id NOT IN (
|
||
SELECT DISTINCT user_id
|
||
FROM itikhomirov.core_fact_lesson_views
|
||
)
|
||
ORDER BY u.user_id;
|
||
|
||
-- Задание 4: Создай витрину course_completion_rate
|
||
|
||
CREATE TABLE itikhomirov.course_completion_rate
|
||
ON CLUSTER clickhouse_cluster
|
||
(
|
||
user_id UInt32,
|
||
user_name String,
|
||
course_id UInt32,
|
||
course_title String,
|
||
lessons_in_course UInt32,
|
||
lessons_viewed UInt32,
|
||
completion_rate Float32
|
||
)
|
||
ENGINE = ReplicatedMergeTree(
|
||
'/clickhouse/tables/{shard}/itikhomirov/course_completion_rate',
|
||
'{replica}'
|
||
)
|
||
ORDER BY (user_id, course_id);
|
||
|
||
|
||
-- Создаем Materialized View
|
||
|
||
CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate
|
||
ON CLUSTER clickhouse_cluster
|
||
TO itikhomirov.course_completion_rate
|
||
AS
|
||
WITH
|
||
lessons_per_course AS (
|
||
SELECT
|
||
course_id,
|
||
countDistinct(lesson_id) AS lessons_in_course
|
||
FROM itikhomirov.core_lesson
|
||
GROUP BY course_id
|
||
),
|
||
views_per_user_course AS (
|
||
SELECT
|
||
user_id,
|
||
course_id,
|
||
countDistinct(lesson_id) AS lessons_viewed
|
||
FROM itikhomirov.core_fact_lesson_views
|
||
GROUP BY user_id, course_id
|
||
)
|
||
SELECT
|
||
u.user_id AS user_id,
|
||
u.name AS user_name,
|
||
c.course_id AS course_id,
|
||
c.title AS course_title,
|
||
lpc.lessons_in_course AS lessons_in_course,
|
||
coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
|
||
ROUND(
|
||
COALESCE(
|
||
NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
|
||
/ NULLIF(lpc.lessons_in_course * 1.0, 0),
|
||
0
|
||
),
|
||
2
|
||
) AS completion_rate
|
||
FROM itikhomirov.core_user AS u
|
||
JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id
|
||
JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
|
||
JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
|
||
LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id
|
||
ORDER BY
|
||
u.user_id,
|
||
c.course_id;
|
||
|
||
-- Вставляем данные в таблицу pet.course_completion_rate
|
||
|
||
INSERT INTO itikhomirov.course_completion_rate
|
||
-- Создаем две временные таблицы:
|
||
-- 1) Сколько уроков в каждом курсе
|
||
WITH
|
||
lessons_per_course AS (
|
||
SELECT
|
||
course_id,
|
||
countDistinct(lesson_id) AS lessons_in_course
|
||
FROM itikhomirov.core_lesson
|
||
GROUP BY course_id
|
||
),
|
||
-- 2) Сколько уроков каждого курса посмотрел пользователь
|
||
views_per_user_course AS (
|
||
SELECT
|
||
user_id,
|
||
course_id,
|
||
countDistinct(lesson_id) AS lessons_viewed
|
||
FROM itikhomirov.core_fact_lesson_views
|
||
GROUP BY user_id, course_id
|
||
)
|
||
-- Основной запрос
|
||
SELECT
|
||
u.user_id AS user_id,
|
||
u.name AS user_name,
|
||
c.course_id AS course_id,
|
||
c.title AS course_title,
|
||
lpc.lessons_in_course AS lessons_in_course,
|
||
coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
|
||
ROUND(
|
||
COALESCE(
|
||
NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
|
||
/ NULLIF(lpc.lessons_in_course * 1.0, 0),
|
||
0
|
||
),
|
||
2
|
||
) AS completion_rate
|
||
FROM itikhomirov.core_user AS u
|
||
JOIN itikhomirov.core_enrollments AS e ON u.user_id = e.user_id
|
||
JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
|
||
JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
|
||
LEFT JOIN views_per_user_course AS vpc ON vpc.user_id = u.user_id AND vpc.course_id = c.course_id
|
||
ORDER BY
|
||
u.user_id,
|
||
c.course_id;
|
||
|
||
-- Этап 3. Анализ и запросы
|
||
|
||
-- Задание 5: Напиши SQL-запросы к витринам
|
||
|
||
-- Уроки с хотя бы одним просмотром
|
||
SELECT
|
||
lesson_title,
|
||
course_title,
|
||
total_views
|
||
FROM itikhomirov.lesson_popularity_summary
|
||
WHERE total_views > 0
|
||
ORDER BY lesson_title ASC;
|
||
|
||
-- Топ‑3 урока внутри одного курса
|
||
SELECT
|
||
lesson_title,
|
||
course_title,
|
||
total_views
|
||
FROM itikhomirov.lesson_popularity_summary
|
||
WHERE course_title = 'SQL для начинающих'
|
||
ORDER BY total_views DESC
|
||
LIMIT 3;
|
||
|
||
-- Пользователи, которые завершили курсы на 100%
|
||
SELECT
|
||
user_id,
|
||
user_name,
|
||
course_title,
|
||
completion_rate
|
||
FROM itikhomirov.course_completion_rate
|
||
WHERE completion_rate = 1
|
||
ORDER BY user_id;
|
||
|
||
-- Этап 4. Тестирование и валидация
|
||
|
||
-- Задание 6: Сравни данные с Greenplum**
|
||
|
||
-- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
|
||
|
||
|
||
-- 2. Сравни с ClickHouse-результатами:
|
||
-- - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
|
||
-- - Проверяй `NULL`, `0`, округления
|
||
|
||
|
||
-- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
|
||
|
||
SELECT
|
||
sum(total_views) AS sum_total_views,
|
||
avg(total_views) AS avg_total_views,
|
||
min(total_views) AS min_total_views,
|
||
max(total_views) AS max_total_views
|
||
FROM itikhomirov.lesson_popularity_summary
|
||
WHERE lesson_id IN (1, 3, 5);
|
||
|
||
SELECT
|
||
sum(total_views) AS sum_total_views,
|
||
avg(total_views) AS avg_total_views,
|
||
min(total_views) AS min_total_views,
|
||
max(total_views) AS max_total_views
|
||
FROM itikhomirov.lesson_popularity_summary
|
||
WHERE lesson_id = 999;
|
||
|
||
-- Отчёт в .doc файле
|
||
|