From 97d55ac2672487c8eb6c074584d25d961e7cee04 Mon Sep 17 00:00:00 2001 From: dr0zd Date: Thu, 29 Jan 2026 04:46:45 +0700 Subject: [PATCH 01/11] OS-314 fixes because I've broken full git struct --- mdrozdov/greenplum/core_layer.sql | 111 ++++++++++++++++++++++++++++++ mdrozdov/greenplum/dm_layer.sql | 71 +++++++++++++++++++ mdrozdov/greenplum/raw_layer.sql | 44 ++++++++++++ 3 files changed, 226 insertions(+) create mode 100644 mdrozdov/greenplum/core_layer.sql create mode 100644 mdrozdov/greenplum/dm_layer.sql create mode 100644 mdrozdov/greenplum/raw_layer.sql diff --git a/mdrozdov/greenplum/core_layer.sql b/mdrozdov/greenplum/core_layer.sql new file mode 100644 index 0000000..557e771 --- /dev/null +++ b/mdrozdov/greenplum/core_layer.sql @@ -0,0 +1,111 @@ +-- Измерение пользователей + CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE + ) + distributed by (user_id); + + -- Измерение курсов + CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT + ) + distributed by (course_id); + + -- Измерение уроков + CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) + ) + distributed by (lesson_id); + + -- Опционально: измерение времени + CREATE TABLE core.dim_date ( + date DATE PRIMARY KEY, + year INT, + month INT, + day INT, + weekday TEXT + ) + distributed replicated; + + -- Таблицы фактов: + CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP + ) + distributed by (course_id); + + CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP + ) + distributed by (course_id); + + --ETL переброс данных из raw в core + INSERT INTO core.dim_user (user_id, name, age, email, registration_date) + SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE + FROM raw.users; + + INSERT INTO core.dim_course (course_id, title, category) + SELECT DISTINCT + id::INT, + title::TEXT, + category::TEXT + FROM raw.courses; + + INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) + SELECT DISTINCT + id::INT, + title::TEXT, + duration_min::INT, + course_id::INT + FROM raw.lessons; + + INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) + SELECT DISTINCT + id::INT, + title::TEXT, + duration_min::INT, + course_id::INT + FROM raw.lessons; + + -- Заполнение таблицы фактов + insert into core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) + select + lw.user_id, + lw.lesson_id, + l.course_id, + lw.viewed_at + from raw.lessons_views lw + join raw.lessons l on lw.lesson_id = l.id; + + insert into core.enrollments (user_id, lesson_id, course_id, viewed_at) + select + lw.user_id, + lw.lesson_id, + l.course_id, + lw.viewed_at + from raw.lessons_views lw + join raw.lessons l on lw.lesson_id = l.id; + + insert into enrollments (user_id, course_id, enrolled_at) + select + user_id, + course_id, + enrolled_at + from raw.enrollments; \ No newline at end of file diff --git a/mdrozdov/greenplum/dm_layer.sql b/mdrozdov/greenplum/dm_layer.sql new file mode 100644 index 0000000..991db90 --- /dev/null +++ b/mdrozdov/greenplum/dm_layer.sql @@ -0,0 +1,71 @@ +--lesson_popularity_summary data mart + create materialized view dm.lesson_popularity_summary as + select + dl.lesson_id, + dl.title as lesson_title, + dl.course_id, + dc.title as course_title, + count(flv.lesson_id) as total_views, + count(distinct flv.user_id) as unique_users, + min(flv.viewed_at) as first_view, + max(flv.viewed_at) as last_view + from core.dim_lesson dl + left join core.dim_course dc + on dl.course_id = dc.course_id + left join core.fact_lesson_views flv + on dl.lesson_id = flv.lesson_id + group by dl.lesson_id, dl.title, dl.course_id, dc.title + + -- inactive_users_summary data mart + create materialized view dm.inactive_users_summary as + select + du.user_id, + du.name as user_name, + du.email, + du.age, + count(e.course_id) as enrollments_count, + du.registration_date + from core.dim_user du + left join core.enrollments e + on du.user_id = e.user_id + left join core.fact_lesson_views flv + on du.user_id = flv.user_id + where flv.user_id is null + group by du.user_id, du.name, du.email, du.age, du.registration_date + + -- course_completion_rate data mart + create materialized view dm.course_completion_rate as + with total_lessons_per_course as( + select + course_id, + count(*) as lessons_in_course + from core.dim_lesson + group by course_id + ), lessons_viewed as ( + select + user_id, + course_id, + count(*) as lessons_viewed + from fact_lesson_views flv + group by user_id, course_id + ) + select + du.user_id as user_id, + du.name as user_name, + dc.course_id as course_id, + dc.title as course_title, + tlpc.lessons_in_course, + coalesce(lv.lessons_viewed, 0) as lessons_viewed, + round( + coalesce( + coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 + ), 2) as completion_rate + from core.enrollments e + join core.dim_user du + on e.user_id = du.user_id + join total_lessons_per_course tlpc + on tlpc.course_id = e.course_id + join core.dim_course dc + on e.course_id = dc.course_id + left join lessons_viewed lv + on du.user_id = lv.user_id and dc.course_id = lv.course_id \ No newline at end of file diff --git a/mdrozdov/greenplum/raw_layer.sql b/mdrozdov/greenplum/raw_layer.sql new file mode 100644 index 0000000..d2ab8c9 --- /dev/null +++ b/mdrozdov/greenplum/raw_layer.sql @@ -0,0 +1,44 @@ +create schema raw; + create schema core; + create schema dm; + + create table raw.users( + id INT, + name varchar(50), + age INT, + email varchar(50), + registration_date date + ) + distributed by (id); + + create table raw.courses ( + id INT, + title text, + category varchar(40), + created_at date + ) + distributed by (id); + + create table raw.lessons ( + id INT, + course_id INT, + title TEXT, + duration_min INT + ) + distributed by (course_id); + + create table raw.enrollments ( + id INT, + user_id INT, + course_id INT, + enrolled_at date + ) + distributed by (user_id); + + create table raw.lessons_views( + id int, + user_id int, + lesson_id int, + viewed_at timestamp + ) + distributed by (user_id) \ No newline at end of file From 0e385bd6d09a82e9df62bfa7816024f6f55e20ac Mon Sep 17 00:00:00 2001 From: dr0zd Date: Thu, 29 Jan 2026 04:50:55 +0700 Subject: [PATCH 02/11] task up to arrays --- mdrozdov/clickhouse/array_map_nested.sql | 23 ++++ mdrozdov/clickhouse/data_types.sql | 61 ++++++++++ mdrozdov/clickhouse/joins.sql | 103 ++++++++++++++++ mdrozdov/clickhouse/merge_tree_family.sql | 64 ++++++++++ mdrozdov/clickhouse/queries.sql | 136 ++++++++++++++++++++++ mdrozdov/clickhouse/view_distribute.sql | 33 ++++++ 6 files changed, 420 insertions(+) create mode 100644 mdrozdov/clickhouse/array_map_nested.sql create mode 100644 mdrozdov/clickhouse/data_types.sql create mode 100644 mdrozdov/clickhouse/joins.sql create mode 100644 mdrozdov/clickhouse/merge_tree_family.sql create mode 100644 mdrozdov/clickhouse/queries.sql create mode 100644 mdrozdov/clickhouse/view_distribute.sql diff --git a/mdrozdov/clickhouse/array_map_nested.sql b/mdrozdov/clickhouse/array_map_nested.sql new file mode 100644 index 0000000..02959e9 --- /dev/null +++ b/mdrozdov/clickhouse/array_map_nested.sql @@ -0,0 +1,23 @@ +--task 1 + select + user_id, + indexOf(tags, 'promo') as promo_index, + arrayJoin(tags) + from events + where has(tags, 'promo') + + -- task 2 + select + user_id, + scores['math'] as math_score, + mapValues(scores) as all_values + from metrics + + -- task 3 + select + name, + contact.type as contact_type, + contact.value as contact_value + from users_nested + array join contacts as contact + where contact.type = 'email' \ No newline at end of file diff --git a/mdrozdov/clickhouse/data_types.sql b/mdrozdov/clickhouse/data_types.sql new file mode 100644 index 0000000..c7089fb --- /dev/null +++ b/mdrozdov/clickhouse/data_types.sql @@ -0,0 +1,61 @@ +-- task 1 + CREATE TABLE raw_data ( + user_id String, + balance String, + created_at String + ) ENGINE = MergeTree ORDER BY tuple(); + + select + user_id, + toDecimal64OrNull(balance, 2) as balance, + toTimeDate64OrNull(created_at, 3) as created_at + from raw_data + where + toDecimal64OrNull(balance, 2) is not null and + toTimeDate64OrNull(created_at, 3) is not null + + -- task 2 (failed) + CREATE TABLE events ( + event_id String, + event_date String, -- форматы: "2023-01-01", "01.02.2023", "invalid" + event_time String -- форматы: "12:30:45", "25:99:99" + ); + + select + event_id, + toStartOfHour(toDateTimeOrNull(concatWithSeparator(' ', events_date, events_time))) + from events + where + toDateOrNull(event_date) is not null and toTimeOrNull(event_time) is not null + + -- task 3 + CREATE TABLE user_activity ( + user_id UInt32, + actions String -- JSON: ["login", "purchase", "logout"] + ); + SELECT + user_id, + actions_arr, + length(actions_arr) AS actions_count + FROM + ( + SELECT + user_id, + JSONExtract(actions, 'Array(String)') AS actions_arr + FROM user_activity + ) + WHERE has(actions_arr, 'purchase'); + + -- task 4 + CREATE TABLE sensor_data ( + sensor_id UInt32, + value Float64, + status Enum('ok' = 1, 'error' = 2) + ); + SELECT + toString(status) as status, + avg(value) as avg_value + from sensor_data + where + toDecimal64OrNull(value, 2) is not null + group by status \ No newline at end of file diff --git a/mdrozdov/clickhouse/joins.sql b/mdrozdov/clickhouse/joins.sql new file mode 100644 index 0000000..9b797cb --- /dev/null +++ b/mdrozdov/clickhouse/joins.sql @@ -0,0 +1,103 @@ +--task 1 + create table mdrozdov.products ( + product_id Int, + name String + ) + engine = MergeTree + order by product_id + + insert into mdrozdov.products values + (101, 'iPhone 14'), + (102, 'Galaxy S22'), + (103, 'Pixel 7') + + create table mdrozdov.prices ( + product_id Int, + price Int, + region String + ) + engine = MergeTree + order by product_id + + insert into mdrozdov.prices values + (101, 999, 'US'), + (101, 1049, 'EU'), + (102, 850, 'US'), + (102, 899, 'CA'), + (103, 799, 'JP') + + -- 1) + select + product_id, + name, + price, + region + from mdrozdov.products + any left join mdrozdov.prices + on mdrozdov.products.product_id = mdrozdov.prices.product_id + + -- 2) + create table mdrozdov.ticks + ON CLUSTER clickhouse_cluster ( + time DateTime, + symbol String, + price Decimal(10, 2) + ) + engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/ticks', + '{replica}' + ) + order by (time, symbol) + + insert into mdrozdov.ticks values + ('2025-07-17 10:00:00', 'AAPL', 190.5), + ('2025-07-17 10:01:00', 'AAPL', 191.2), + ('2025-07-17 10:02:30', 'AAPL', 192.0), + ('2025-07-17 10:05:00', 'GOOG', 2850.0) + + create table mdrozdov.orders + ON CLUSTER clickhouse_cluster ( + time DateTime, + symbol String, + size Int + ) + engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/orders', + '{replica}' + ) + order by size + + insert into mdrozdov.orders values + ('2025-07-17 10:00:30', 'AAPL', 50), + ('2025-07-17 10:02:00', 'AAPL', 75), + ('2025-07-17 10:03:00', 'AAPL', 20), + ('2025-07-17 10:04:00', 'GOOG', 15) + + select + o.time, + o.symbol, + t.price as last_price + from mdrozdov.orders o + asof join mdrozdov.ticks t + on o.symbol = t.symbol and o.time >= t.time + + -- task 3 + select + user_id, + name + from users + where user_id in ( + select user_id + from payments + where amount > 1000 + ) + + -- task 4 + select + bl.user_id, + u.name, + bl.event_time, + bl.event_type, + from big_logs bl + any left join users u + using(user_id) \ No newline at end of file diff --git a/mdrozdov/clickhouse/merge_tree_family.sql b/mdrozdov/clickhouse/merge_tree_family.sql new file mode 100644 index 0000000..71db981 --- /dev/null +++ b/mdrozdov/clickhouse/merge_tree_family.sql @@ -0,0 +1,64 @@ +--page_views table + create table page_views ( + user_id UInt64, + url String, + view_time DateTime + ) + Engine = MergeTree + Order by (user_id, view_time) + Partition by toYYYYMM(view_time) + + -- users table + create table users( + id UInt64, + name String, + updated_at DateTime + ) + ENGINE = ReplacingMergeTree(updated_at) + Order by id; + + -- region_sales table + create table region_sales( + region String, + date Date, + sales UInt64 + ) + ENGINE = SummingMergeTree + Partition by toYYYYMM(date) + Order by (region, date) + + insert into mdrozdov.region_sales (region, date, sales) + values ('Russia', '2025-01-14', 30), ('Russia', '2025-01-16', 22), ('USA', '2025-01-14', 1400) + OPTIMIZE TABLE mdrozdov.region_sales FINAL + + -- user_metrics table + create table mdrozdov.user_metrics ( + region String, + avg_age_state AggregateFunction(avg, UInt8), + active_users_state AggregateFunction(uniq, UInt64) + ) + Engine = AggregatingMergeTree + Order by region + + INSERT INTO mdrozdov.user_metrics + SELECT + 'Moscow' AS region, + avgState(toUInt8(25)) AS avg_age_state, + uniqState(toUInt64(100)) AS active_users_state + UNION ALL + SELECT + 'Moscow', + avgState(toUInt8(30)), + uniqState(toUInt64(50)) + UNION ALL + SELECT + 'SPB', + avgState(toUInt8(20)), + uniqState(toUInt64(10)); + + SELECT + region, + avgMerge(avg_age_state) AS avg_age, + uniqMerge(active_users_state) AS active_users + FROM mdrozdov.user_metrics + GROUP BY region \ No newline at end of file diff --git a/mdrozdov/clickhouse/queries.sql b/mdrozdov/clickhouse/queries.sql new file mode 100644 index 0000000..572ace7 --- /dev/null +++ b/mdrozdov/clickhouse/queries.sql @@ -0,0 +1,136 @@ +CREATE TABLE users ( + user_id Int, + name String, + age Int, + country String, + is_active bool + ) + engine = MergeTree + order by user_id + + insert into mdrozdov.users values + (1, 'Ivan' ,30, 'RU', 1), + (2, 'Anna' ,25, 'RU', 1), + (3, 'Petr' ,45, 'RU', 0), + (4, 'Olga' ,35, 'RU', 1), + (5, 'John' ,55, 'US', 1), + (6, 'Alice' ,42, 'US', 0), + (7, 'Tom' ,38, 'US', 1), + (8, 'Lucas' ,28, 'BR', 1), + (9, 'Maria' ,60, 'BR', 1), + (10, 'João' ,27, 'BR', 0), + (11, 'Kenji' ,33, 'JP', 1), + (12, 'Yuki' ,29, 'JP', 0), + (13, 'Chen' ,40, 'CN', 1), + (14, 'Li' ,48, 'CN', 1), + (15, 'Fatima',31, 'AE', 1), + (16, 'Ahmed' ,36, 'AE', 1), + (17, 'George',24, 'US', 1), + (18, 'Nina' ,34, 'RU', 1), + (19, 'Max' ,50, 'DE', 1), + (20, 'Sophie',22, 'DE', 0) + + --1. Получи 10 самых старших **активных** пользователей. + select + user_id, + name, + age, + country, + is_active + from mdrozdov.users + where is_active + order by age desc + limit 10 + --2. Выведи пользователей из `country = 'RU'`, отсортированных по имени. + select + user_id, + name, + age, + country, + is_active + from mdrozdov.users + where country = 'RU' + order by name asc + --3. Что произойдёт, если написать `LIMIT 10` **без `ORDER BY`**? + -- ответ будет недетерминирован + --4. Посчитай количество пользователей по странам. + select + country, + count(user_id) as total_users + from mdrozdov.users + group by country + --5. Для каждой страны выведи минимальный и максимальный возраст. + select + country, + min(age) as min_user_age, + max(age) as max_user_age + from mdrozdov.users + group by country + --6. Отсортируй страны по убыванию количества пользователей. + select + country, + count(user_id) as total_users + from mdrozdov.users + group by country + order by total_users desc + + -- task 2 + CREATE TABLE mdrozdov.events ( + user_id Int, + event_time DateTime, + event_type String --mb better enum but idc + ) + engine = MergeTree + order by user_id + insert into events values + (1, '2025-07-17 09:00:00', 'click'), + (1, '2025-07-17 09:05:00', 'view'), + (1, '2025-07-17 09:10:00', 'purchase'), + (2, '2025-07-17 10:00:00', 'click'), + (2, '2025-07-17 10:05:00', 'click'), + (2, '2025-07-17 11:00:00', 'view'), + (3, '2025-07-16 12:00:00', 'view'), + (3, '2025-07-16 12:30:00', 'click'), + (3, '2025-07-17 08:00:00', 'purchase'), + (4, '2025-07-17 13:00:00', 'click'), + (4, '2025-07-17 14:00:00', 'view'), + (4, '2025-07-17 15:00:00', 'view'), + (5, '2025-07-16 20:00:00', 'view'), + (5, '2025-07-17 01:00:00', 'click'), + (5, '2025-07-17 06:00:00', 'purchase'), + (6, '2025-07-17 07:30:00', 'view'), + (6, '2025-07-17 08:45:00', 'click'), + (7, '2025-07-16 23:59:59', 'click'), + (7, '2025-07-17 00:01:00', 'purchase'), + (7, '2025-07-17 10:10:00', 'view') + --1-2) Добавь колонку с кумулятивным количеством событий по user_id (используй count() OVER ...) + select + user_id, + event_time, + event_type, + count(*) over (partition by user_id order by event_time) as total_click_exact_moment, + row_number() over (partition by user_id order by event_time) as row_number + from mdrozdov.events + --3) да, поддерживается, но не обязательно для коммулятивного суммирования строк + --4-6) Сделай WITH recent_events AS (...) — все события за последние 24 часа. + with recent_events AS ( + select * + from mdrozdov.events + where event_time >= '2025-07-18'::DATE - interval '1 day' + ), category_list as ( + select 'click' as event_type, 'interaction' as category + union all + select 'view', 'interaction' + union all + select 'purchase', 'conversion' + ) + select + user_id, + count(*) as total_events_last_24h, + count(distinct cl.category) as uniq_categories + from recent_events re + left join category_list cl + using(event_type) + group by user_id + order by total_events_last_24h desc, user_id asc + limit 5 \ No newline at end of file diff --git a/mdrozdov/clickhouse/view_distribute.sql b/mdrozdov/clickhouse/view_distribute.sql new file mode 100644 index 0000000..d330749 --- /dev/null +++ b/mdrozdov/clickhouse/view_distribute.sql @@ -0,0 +1,33 @@ +-- task 1 + CREATE TABLE dist_table + ENGINE = Distributed( + 'cluster_name', -- нет доступа к конфигу infra clickhouse + 'mdrozdov', + 'logs', + rand() + ); + --rand() это параметр по которому мы распределяем данные при insert на разные ноды, в данном + --случае нам это нужно для равномерности распределения, потому что таблица не требует особых связей или тп + + -- task 2 + --Создай VIEW top_active_users, который показывает топ-100 пользователей по числу событий + CREATE VIEW top_active_users AS + select user_id, count(*) as total_events + from events + group by user_id + order by total_events desc + limit 100 + -- не подходит для дашборда с миллионами строк, потому что вычисляется каждый раз при вызове + -- для дашбордов лучше подойдет materialized view, тк хранит данные запроса и обновляется автоматически + -- при вставке новых данных + + --task 3 + CREATE MATERIALIZED VIEW + ENGINE = SummingMergeTree + order by date + mv_clicks_per_day AS + select + toDate(event_time) as date, + count(*) as total_clicks + from raw_events + group date \ No newline at end of file From 341302725da2074f12fc0ab49e2e9952f5da093d Mon Sep 17 00:00:00 2001 From: dr0zd Date: Thu, 29 Jan 2026 20:36:44 +0700 Subject: [PATCH 03/11] partitions solutions --- mdrozdov/clickhouse/partitions.sql | 98 ++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 mdrozdov/clickhouse/partitions.sql diff --git a/mdrozdov/clickhouse/partitions.sql b/mdrozdov/clickhouse/partitions.sql new file mode 100644 index 0000000..817d95b --- /dev/null +++ b/mdrozdov/clickhouse/partitions.sql @@ -0,0 +1,98 @@ +-- task 1 +create table mdrozdov.sales2 +on CLUSTER clickhouse_cluster ( + id Int, + region String, + sale_date Date, + amount Decimal(8, 2) +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/sales2', + '{replica}' + ) +partition by toYYYYMM(sale_date) +order by id +-- удаление таблицы +-- drop table mdrozdov.sales +-- on CLUSTER clickhouse_cluster; не удаляет таблицу полностью, она остается в zookeeper +INSERT into mdrozdov.sales2 values +(1, 'RU', '2024-05-01', 100.0), +(2, 'RU', '2024-05-15', 120.0), --эти значения удалились +(3, 'US', '2024-06-01', 90.0), +(4, 'US', '2024-07-03', 150.0), +(5, 'DE', '2024-07-04', 160.0) +alter table mdrozdov.sales2 +drop partition '202405' +--2 значения удалились, которые хранились в соответствующей партиции + +-- task 2 +create table mdrozdov.events2 +on CLUSTER clickhouse_cluster ( + id Int, + event_type String, + event_date Date +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/events2', + '{replica}' + ) +partition by event_type +order by id + +INSERT into mdrozdov.events2 values +(1, 'login', '2024-06-01'), +(2, 'click', '2024-06-01'), +(3, 'click', '2024-06-02'), +(4, 'purchase', '2024-06-03'), +(5, 'login', '2024-06-04') + +alter table mdrozdov.events2 +drop partition 'click' + +-- task 3 +create table mdrozdov.logs +on CLUSTER clickhouse_cluster ( + id Int, + user_id Int, + log_time DateTime +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/logs', + '{replica}' + ) +partition by cityHash64(user_id) % 100 +order by id + +insert into mdrozdov.logs values +(1, 123, '2024-07-17 10:00:00'), +(2, 456, '2024-07-17 10:01:00'), +(3, 789, '2024-07-17 10:02:00') +--Что ты выиграешь с точки зрения баланса? +--равномерное распределение данных в разных партициях, лучшее управление обновлением и удалением данных +-- Почему такой тип PARTITION не ускорит SELECT * WHERE user_id = 456? +-- потому что clickhouse не умеет выполнять условия партиции, чтобы найти сразу ту в которой надо искать, +-- поэтому будет пробегать все 100, чтобы найти нужное, поэтому важно, чтобы таблица была отсортирована по user_id + +-- task 4 +create table mdrozdov.visits +on CLUSTER clickhouse_cluster ( + id Int, + region String, + visit_date Date +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/visits', + '{replica}' + ) +partition by (region, toYYYYMM(visit_date)) +order by (region, visit_date) + +insert into mdrozdov.visits values +(1, 'RU', '2024-06-01'), +(2, 'RU', '2024-06-15'), +(3, 'US', '2024-06-03'), +(4, 'US', '2024-07-01'), +(5, 'RU', '2024-07-05') + +alter table mdrozdov.visits +drop partition ('RU', 202407) -- эффективное удаление июльских данных по россии \ No newline at end of file From eca39dbceedc6afbb5f304cccd6d9d22dcf0222a Mon Sep 17 00:00:00 2001 From: dr0zd Date: Thu, 29 Jan 2026 23:22:44 +0700 Subject: [PATCH 04/11] indexes --- mdrozdov/clickhouse/secondary_index.sql | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 mdrozdov/clickhouse/secondary_index.sql diff --git a/mdrozdov/clickhouse/secondary_index.sql b/mdrozdov/clickhouse/secondary_index.sql new file mode 100644 index 0000000..3a902d7 --- /dev/null +++ b/mdrozdov/clickhouse/secondary_index.sql @@ -0,0 +1,65 @@ +--task 1 +create table mdrozdov.logs3 +on CLUSTER clickhouse_cluster ( + log_id Int, + ts DateTime +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/logs3', + '{replica}' + ) +order by ts + +alter table mdrozdov.logs3 +add index idx_ts ts type minmax GRANULARITY 2; + +select * +from mdrozdov.logs3 +where ts between '2024-06-02' and '2024-06-03' + +SELECT + query, + read_rows, + result_rows, + read_bytes +FROM system.query_log +WHERE query LIKE '%logs%' +ORDER BY event_time DESC +LIMIT 1; + +-- task 2 +create table mdrozdov.clicks +on CLUSTER clickhouse_cluster ( + id UInt64, + country String +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/clicks', + '{replica}' + ) +order by id + +alter table mdrozdov.clicks +add index idx_country type set(100) GRANULARITY 2 + +Explain + select * + from mdrozdov.clicks + where country = 'RU' + +-- task 3 +create table mdrozdov.users +on CLUSTER clickhouse_cluster ( + id UInt64, + email String +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/users', + '{replica}' + ) +order by id + +ALTER TABLE users +ADD INDEX idx_email email TYPE bloom_filter(0.01) GRANULARITY 2; + +SELECT * FROM users WHERE email = 'alice@mail.com'; \ No newline at end of file From 73988517db77f588dd09b1aa7c404aa7672b8433 Mon Sep 17 00:00:00 2001 From: dr0zd Date: Sat, 31 Jan 2026 01:16:20 +0700 Subject: [PATCH 05/11] stage 1 tables --- mdrozdov/clickhouse/stage1.sql | 96 ++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 mdrozdov/clickhouse/stage1.sql diff --git a/mdrozdov/clickhouse/stage1.sql b/mdrozdov/clickhouse/stage1.sql new file mode 100644 index 0000000..232cbac --- /dev/null +++ b/mdrozdov/clickhouse/stage1.sql @@ -0,0 +1,96 @@ +-- удалить таблицу если раньше создавали +DROP TABLE IF EXISTS mdrozdov.core_fact_lesson_views +ON CLUSTER clickhouse_cluster; + +-- таблицы измерений +-- user dimention +CREATE TABLE mdrozdov.core_dim_user +ON CLUSTER clickhouse_cluster +( + user_id Int32, + name String, + age Int32, + email String, + registration_date Date +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_user', + '{replica}' +) +ORDER BY user_id; + +-- course dimention +CREATE TABLE mdrozdov.core_dim_course +ON CLUSTER clickhouse_cluster +( + course_id Int32, + title String, + category String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_course', + '{replica}' +) +ORDER BY course_id; + +-- lesson dimention +CREATE TABLE mdrozdov.core_dim_lesson +ON CLUSTER clickhouse_cluster +( + lesson_id Int32, + title String, + duration_min Int32, + course_id String, +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_lesson', + '{replica}' +) +ORDER BY course_id; + +-- date dimention +CREATE TABLE mdrozdov.core_dim_date +ON CLUSTER clickhouse_cluster +( + date DATE, + year Int32, + month Int32, + day Int32, + weekday String +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_dim_date', + '{replica}' +) +ORDER BY date; + + +-- таблицы фактов +-- core_enrollments +CREATE TABLE mdrozdov.core_enrollments +ON CLUSTER clickhouse_cluster +( + user_id Int32, + course_id Int32, + enrolled_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_enrollments', + '{replica}' +) +ORDER BY course_id; + +-- core_fact_lesson_views +CREATE TABLE mdrozdov.core_fact_lesson_views +ON CLUSTER clickhouse_cluser +( + user_id Int32, + lesson_id Int32, + course_id Int32, + viewed_at DateTime64(3) +) +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/core_fact_lesson_views', + '{replica}' +) +ORDER BY course_id; From 0433c728282ca3f381a92aadda765cf9cf2f1402 Mon Sep 17 00:00:00 2001 From: dr0zd Date: Sat, 31 Jan 2026 03:29:26 +0700 Subject: [PATCH 06/11] fix comments --- mdrozdov/clickhouse/data_types.sql | 8 ++++---- mdrozdov/clickhouse/joins.sql | 4 ++-- mdrozdov/clickhouse/merge_tree_family.sql | 2 +- mdrozdov/clickhouse/queries.sql | 6 +++--- mdrozdov/clickhouse/secondary_index.sql | 6 +++--- mdrozdov/clickhouse/view_distribute.sql | 3 ++- mdrozdov/greenplum/core_layer.sql | 19 +------------------ 7 files changed, 16 insertions(+), 32 deletions(-) diff --git a/mdrozdov/clickhouse/data_types.sql b/mdrozdov/clickhouse/data_types.sql index c7089fb..fc580c3 100644 --- a/mdrozdov/clickhouse/data_types.sql +++ b/mdrozdov/clickhouse/data_types.sql @@ -8,11 +8,11 @@ select user_id, toDecimal64OrNull(balance, 2) as balance, - toTimeDate64OrNull(created_at, 3) as created_at + toDateTime64OrNull(created_at, 3) as created_at from raw_data where toDecimal64OrNull(balance, 2) is not null and - toTimeDate64OrNull(created_at, 3) is not null + toDateTime64OrNull(created_at, 3) is not null -- task 2 (failed) CREATE TABLE events ( @@ -23,7 +23,7 @@ select event_id, - toStartOfHour(toDateTimeOrNull(concatWithSeparator(' ', events_date, events_time))) + toStartOfHour(toDateTimeOrNull(concatWithSeparator(' ', event_date, event_time))) from events where toDateOrNull(event_date) is not null and toTimeOrNull(event_time) is not null @@ -57,5 +57,5 @@ avg(value) as avg_value from sensor_data where - toDecimal64OrNull(value, 2) is not null + value is not null group by status \ No newline at end of file diff --git a/mdrozdov/clickhouse/joins.sql b/mdrozdov/clickhouse/joins.sql index 9b797cb..4701740 100644 --- a/mdrozdov/clickhouse/joins.sql +++ b/mdrozdov/clickhouse/joins.sql @@ -65,7 +65,7 @@ '/clickhouse/tables/{shard}/mdrozdov/orders', '{replica}' ) - order by size + order by (symbol, time) insert into mdrozdov.orders values ('2025-07-17 10:00:30', 'AAPL', 50), @@ -97,7 +97,7 @@ bl.user_id, u.name, bl.event_time, - bl.event_type, + bl.event_type from big_logs bl any left join users u using(user_id) \ No newline at end of file diff --git a/mdrozdov/clickhouse/merge_tree_family.sql b/mdrozdov/clickhouse/merge_tree_family.sql index 71db981..74dbfba 100644 --- a/mdrozdov/clickhouse/merge_tree_family.sql +++ b/mdrozdov/clickhouse/merge_tree_family.sql @@ -18,7 +18,7 @@ Order by id; -- region_sales table - create table region_sales( + create table mdrozdov.region_sales( region String, date Date, sales UInt64 diff --git a/mdrozdov/clickhouse/queries.sql b/mdrozdov/clickhouse/queries.sql index 572ace7..e2d7afe 100644 --- a/mdrozdov/clickhouse/queries.sql +++ b/mdrozdov/clickhouse/queries.sql @@ -1,4 +1,4 @@ -CREATE TABLE users ( +CREATE TABLE mdrozdov.users ( user_id Int, name String, age Int, @@ -113,10 +113,10 @@ CREATE TABLE users ( from mdrozdov.events --3) да, поддерживается, но не обязательно для коммулятивного суммирования строк --4-6) Сделай WITH recent_events AS (...) — все события за последние 24 часа. - with recent_events AS ( + with mdrozdov.recent_events AS ( select * from mdrozdov.events - where event_time >= '2025-07-18'::DATE - interval '1 day' + where event_time >= toDateTime('2025-07-18') - interval 1 day ), category_list as ( select 'click' as event_type, 'interaction' as category union all diff --git a/mdrozdov/clickhouse/secondary_index.sql b/mdrozdov/clickhouse/secondary_index.sql index 3a902d7..53f172c 100644 --- a/mdrozdov/clickhouse/secondary_index.sql +++ b/mdrozdov/clickhouse/secondary_index.sql @@ -40,7 +40,7 @@ engine = ReplicatedMergeTree( order by id alter table mdrozdov.clicks -add index idx_country type set(100) GRANULARITY 2 +add index idx_country country type set(100) GRANULARITY 2 Explain select * @@ -59,7 +59,7 @@ engine = ReplicatedMergeTree( ) order by id -ALTER TABLE users +ALTER TABLE mdrozdov.users ADD INDEX idx_email email TYPE bloom_filter(0.01) GRANULARITY 2; -SELECT * FROM users WHERE email = 'alice@mail.com'; \ No newline at end of file +SELECT * FROM mdrozdov.users WHERE email = 'alice@mail.com'; \ No newline at end of file diff --git a/mdrozdov/clickhouse/view_distribute.sql b/mdrozdov/clickhouse/view_distribute.sql index d330749..41ed079 100644 --- a/mdrozdov/clickhouse/view_distribute.sql +++ b/mdrozdov/clickhouse/view_distribute.sql @@ -23,6 +23,7 @@ --task 3 CREATE MATERIALIZED VIEW + to ENGINE = SummingMergeTree order by date mv_clicks_per_day AS @@ -30,4 +31,4 @@ toDate(event_time) as date, count(*) as total_clicks from raw_events - group date \ No newline at end of file + group by date \ No newline at end of file diff --git a/mdrozdov/greenplum/core_layer.sql b/mdrozdov/greenplum/core_layer.sql index 557e771..8ff3a53 100644 --- a/mdrozdov/greenplum/core_layer.sql +++ b/mdrozdov/greenplum/core_layer.sql @@ -76,14 +76,6 @@ course_id::INT FROM raw.lessons; - INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) - SELECT DISTINCT - id::INT, - title::TEXT, - duration_min::INT, - course_id::INT - FROM raw.lessons; - -- Заполнение таблицы фактов insert into core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) select @@ -94,16 +86,7 @@ from raw.lessons_views lw join raw.lessons l on lw.lesson_id = l.id; - insert into core.enrollments (user_id, lesson_id, course_id, viewed_at) - select - lw.user_id, - lw.lesson_id, - l.course_id, - lw.viewed_at - from raw.lessons_views lw - join raw.lessons l on lw.lesson_id = l.id; - - insert into enrollments (user_id, course_id, enrolled_at) + insert into core.enrollments (user_id, course_id, enrolled_at) select user_id, course_id, From e5d51559c9396da7cbed063e3f5d7475ca8d1897 Mon Sep 17 00:00:00 2001 From: dr0zd Date: Mon, 2 Feb 2026 02:18:43 +0700 Subject: [PATCH 07/11] WIP: pet-project clickhouse MVs --- mdrozdov/clickhouse/stage2.sql | 123 +++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 mdrozdov/clickhouse/stage2.sql diff --git a/mdrozdov/clickhouse/stage2.sql b/mdrozdov/clickhouse/stage2.sql new file mode 100644 index 0000000..c58541c --- /dev/null +++ b/mdrozdov/clickhouse/stage2.sql @@ -0,0 +1,123 @@ +--lesson_popularity_summary data mart +create table mdrozdov.dm_lesson_popularity_summary +ON CLUSTER clickhouse_cluster( + lesson_id UInt32, + lesson_title String, + course_id UInt32, + course_title String, + total_views AggregateFunction(count, UInt64), + unique_users AggregateFunction(uniqExact, Int32), + first_view AggregateFunction(min, DateTime), + last_view AggregateFunction(max, DateTime) +) +ENGINE = ReplicatedAggregatingMergeTree( +'/clickhouse/tables/{shard}/mdrozdov/dm_lesson_popularity_summary', +'{replica}' +) +ORDER BY (lesson_id, course_id); + +create materialized view mdrozdov.lesson_popularity_summary to mdrozdov.dm_lesson_popularity_summary as +select + dl.lesson_id as lesson_id, + dl.title as lesson_title, + dl.course_id as course_id, + dc.title as course_title, + countState(flv.lesson_id) as total_views, + uniqExactState(flv.user_id) as unique_users, + minState(toDateTime(flv.viewed_at)) as first_view, + maxState(toDateTime(flv.viewed_at)) as last_view +from mdrozdov.core_fact_lesson_views flv +left join mdrozdov.core_dim_lesson dl + on dl.lesson_id = flv.lesson_id +left join mdrozdov.core_dim_course dc + on flv.course_id = dc.course_id +group by dl.lesson_id, dl.title, dl.course_id, dc.title + + +--inactive_users_summary +create table mdrozdov.dm_inactive_users_summary +ON CLUSTER clickhouse_cluster( + user_id Int32, + name String, + email Nullable(String), + age Int32, + registered_courses_count AggregateFunction(uniqExact, Int32), + registration_date Date +) +ENGINE = ReplicatedAggregatingMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/dm_inactive_users_summary', + '{replica}' +) +ORDER BY user_id + +create materialized view mdrozdov.inactive_users_summary + to mdrozdov.dm_inactive_users_summary + as + select + du.user_id as user_id, + du.name as name, + du.email as email, + du.age as age, + uniqExactState(e.course_id) as registered_courses_count, + du.registration_date + from mdrozdov.core_fact_lesson_views flv + right join mdrozdov.core_dim_user du + on du.user_id = flv.user_id + left join mdrozdov.core_enrollments e + on du.user_id = e.user_id + where flv.user_id is null + group by du.user_id, du.name, du.email, du.age, du.registration_date + +-- course_completion_rate +create table mdrozdov.dm_course_completion_rate +ON CLUSTER clickhouse_cluster( + user_id Int32, + user_name String, + course_id Int32, + course_title String, + lessons_in_course AggregateFunction(count, UInt64), + lessons_viewed AggregateFunction(count, UInt64), + completion_rate Decimal(4, 3) +) +ENGINE = ReplicatedAggregatingMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/dm_course_completion_rate', + '{replica}' +) +ORDER BY (user_id, course_id) + +create materialized view mdrozdov.course_completion_rate + to mdrozdov.dm_course_completion_rate as + with total_lessons_per_course as( + select + course_id, + count(*) as lessons_in_course + from core.dim_lesson + group by course_id + ), lessons_viewed as ( + select + user_id, + course_id, + count(*) as lessons_viewed + from fact_lesson_views flv + group by user_id, course_id + ) + select + du.user_id as user_id, + du.name as user_name, + dc.course_id as course_id, + dc.title as course_title, + tlpc.lessons_in_course as lessons_in_course, + coalesce(lv.lessons_viewed, 0) as lessons_viewed, + round( + coalesce( + coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 + ), 2) as completion_rate + from mdrozdov.core_enrollments e + join mdrozdov.core_dim_user du + on e.user_id = du.user_id + join total_lessons_per_course tlpc + on tlpc.course_id = e.course_id + join mdrozdov.core_dim_course dc + on e.course_id = dc.course_id + left join lessons_viewed lv + on du.user_id = lv.user_id and dc.course_id = lv.course_id \ No newline at end of file From 3a71777fdde6b27921e543a3fec163c4744f3acb Mon Sep 17 00:00:00 2001 From: dr0zd Date: Wed, 4 Feb 2026 18:47:38 +0700 Subject: [PATCH 08/11] fixed comments stage1 pet-project clickhouse --- mdrozdov/clickhouse/stage1.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mdrozdov/clickhouse/stage1.sql b/mdrozdov/clickhouse/stage1.sql index 232cbac..bc57c30 100644 --- a/mdrozdov/clickhouse/stage1.sql +++ b/mdrozdov/clickhouse/stage1.sql @@ -3,7 +3,7 @@ DROP TABLE IF EXISTS mdrozdov.core_fact_lesson_views ON CLUSTER clickhouse_cluster; -- таблицы измерений --- user dimention +-- user dimension CREATE TABLE mdrozdov.core_dim_user ON CLUSTER clickhouse_cluster ( @@ -19,7 +19,7 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY user_id; --- course dimention +-- course dimension CREATE TABLE mdrozdov.core_dim_course ON CLUSTER clickhouse_cluster ( @@ -33,22 +33,22 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY course_id; --- lesson dimention +-- lesson dimension CREATE TABLE mdrozdov.core_dim_lesson ON CLUSTER clickhouse_cluster ( lesson_id Int32, title String, duration_min Int32, - course_id String, + course_id Int32, ) ENGINE = ReplicatedMergeTree( - '/clickhouse/tables/{shard}/mdrozdov/core_dim_lesson', + '/clickhouse/tables/{shard}/mdrozdov/core_dim_lesson1', --добавил 1 потому что zookeeper ругается на удаление на кластере, что данные есть уже '{replica}' ) ORDER BY course_id; --- date dimention +-- date dimension CREATE TABLE mdrozdov.core_dim_date ON CLUSTER clickhouse_cluster ( @@ -82,7 +82,7 @@ ORDER BY course_id; -- core_fact_lesson_views CREATE TABLE mdrozdov.core_fact_lesson_views -ON CLUSTER clickhouse_cluser +ON CLUSTER clickhouse_cluster ( user_id Int32, lesson_id Int32, From b5f9e9724e714356c24ba7f9b960ba3d731e77e5 Mon Sep 17 00:00:00 2001 From: dr0zd Date: Thu, 5 Feb 2026 01:44:56 +0700 Subject: [PATCH 09/11] fixed core_dim_lesson table primary key --- mdrozdov/clickhouse/stage1.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mdrozdov/clickhouse/stage1.sql b/mdrozdov/clickhouse/stage1.sql index bc57c30..5242dea 100644 --- a/mdrozdov/clickhouse/stage1.sql +++ b/mdrozdov/clickhouse/stage1.sql @@ -43,10 +43,10 @@ ON CLUSTER clickhouse_cluster course_id Int32, ) ENGINE = ReplicatedMergeTree( - '/clickhouse/tables/{shard}/mdrozdov/core_dim_lesson1', --добавил 1 потому что zookeeper ругается на удаление на кластере, что данные есть уже + '/clickhouse/tables/{shard}/mdrozdov/core_dim_lesson2', '{replica}' ) -ORDER BY course_id; +ORDER BY (course_id, lesson_id); -- date dimension CREATE TABLE mdrozdov.core_dim_date From ee22e5a28d42016fbbaf6cadc5d589d0dde2b3fb Mon Sep 17 00:00:00 2001 From: dr0zd Date: Thu, 5 Feb 2026 23:24:49 +0700 Subject: [PATCH 10/11] full solve, but still dont work --- mdrozdov/clickhouse/stage2.sql | 86 +++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/mdrozdov/clickhouse/stage2.sql b/mdrozdov/clickhouse/stage2.sql index c58541c..1c74ed8 100644 --- a/mdrozdov/clickhouse/stage2.sql +++ b/mdrozdov/clickhouse/stage2.sql @@ -1,51 +1,51 @@ --lesson_popularity_summary data mart create table mdrozdov.dm_lesson_popularity_summary ON CLUSTER clickhouse_cluster( - lesson_id UInt32, + lesson_id Int32, lesson_title String, - course_id UInt32, + course_id Int32, course_title String, - total_views AggregateFunction(count, UInt64), - unique_users AggregateFunction(uniqExact, Int32), - first_view AggregateFunction(min, DateTime), - last_view AggregateFunction(max, DateTime) + total_views Int32, + unique_users Int32, + first_view DateTime64(3), + last_view DateTime64(3) ) -ENGINE = ReplicatedAggregatingMergeTree( -'/clickhouse/tables/{shard}/mdrozdov/dm_lesson_popularity_summary', -'{replica}' +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/dm_lesson_popularity_summary1', + '{replica}' ) ORDER BY (lesson_id, course_id); -create materialized view mdrozdov.lesson_popularity_summary to mdrozdov.dm_lesson_popularity_summary as +create materialized view mdrozdov.lesson_popularity_summary +to mdrozdov.dm_lesson_popularity_summary as select - dl.lesson_id as lesson_id, - dl.title as lesson_title, - dl.course_id as course_id, - dc.title as course_title, - countState(flv.lesson_id) as total_views, - uniqExactState(flv.user_id) as unique_users, - minState(toDateTime(flv.viewed_at)) as first_view, - maxState(toDateTime(flv.viewed_at)) as last_view + dl.lesson_id as lesson_id, + dl.title as lesson_title, + dl.course_id as course_id, + dc.title as course_title, + count(flv.lesson_id) as total_views, + count(distinct flv.user_id) as unique_users, + min(flv.viewed_at) as first_view, + max(flv.viewed_at) as last_view from mdrozdov.core_fact_lesson_views flv +right join mdrozdov.core_dim_course dc + on flv.course_id = dc.course_id left join mdrozdov.core_dim_lesson dl on dl.lesson_id = flv.lesson_id -left join mdrozdov.core_dim_course dc - on flv.course_id = dc.course_id group by dl.lesson_id, dl.title, dl.course_id, dc.title - --inactive_users_summary create table mdrozdov.dm_inactive_users_summary ON CLUSTER clickhouse_cluster( user_id Int32, name String, - email Nullable(String), + email String, age Int32, - registered_courses_count AggregateFunction(uniqExact, Int32), + registered_courses_count Int32, registration_date Date ) -ENGINE = ReplicatedAggregatingMergeTree( - '/clickhouse/tables/{shard}/mdrozdov/dm_inactive_users_summary', +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/dm_inactive_users_summary1', '{replica}' ) ORDER BY user_id @@ -54,12 +54,12 @@ create materialized view mdrozdov.inactive_users_summary to mdrozdov.dm_inactive_users_summary as select - du.user_id as user_id, - du.name as name, - du.email as email, - du.age as age, - uniqExactState(e.course_id) as registered_courses_count, - du.registration_date + du.user_id as user_id, + du.name as name, + du.email as email, + du.age as age, + count(e.course_id) as registered_courses_count, + du.registration_date as registration_date from mdrozdov.core_fact_lesson_views flv right join mdrozdov.core_dim_user du on du.user_id = flv.user_id @@ -75,11 +75,11 @@ ON CLUSTER clickhouse_cluster( user_name String, course_id Int32, course_title String, - lessons_in_course AggregateFunction(count, UInt64), - lessons_viewed AggregateFunction(count, UInt64), + lessons_in_course Int32, + lessons_viewed Int32, completion_rate Decimal(4, 3) ) -ENGINE = ReplicatedAggregatingMergeTree( +ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/mdrozdov/dm_course_completion_rate', '{replica}' ) @@ -91,14 +91,14 @@ create materialized view mdrozdov.course_completion_rate select course_id, count(*) as lessons_in_course - from core.dim_lesson + from mdrozdov.core_dim_lesson group by course_id ), lessons_viewed as ( select user_id, course_id, count(*) as lessons_viewed - from fact_lesson_views flv + from mdrozdov.core_fact_lesson_views flv group by user_id, course_id ) select @@ -120,4 +120,16 @@ create materialized view mdrozdov.course_completion_rate join mdrozdov.core_dim_course dc on e.course_id = dc.course_id left join lessons_viewed lv - on du.user_id = lv.user_id and dc.course_id = lv.course_id \ No newline at end of file + on du.user_id = lv.user_id and dc.course_id = lv.course_id + +--данные из-за несовпадения названий столбцов в импорте/экспорте пришлось удалять, короче проще вручную +-- их добавить +insert into mdrozdov.core_fact_lesson_views(user_id, lesson_id, course_id, viewed_at) values +(1, 1, 1, '2023-01-16 10:00:00'), +(1, 2, 1, '2023-01-16 10:15:00'), +(2, 1, 1, '2023-01-21 09:00:00'), +(3, 3, 2, '2023-03-06 12:00:00'), +(4, 4, 3, '2023-04-02 10:00:00'), +(4, 5, 3, '2023-04-03 10:00:00') + +-- данные в mdrozdov.core_enrollments загружал через импорт \ No newline at end of file From bb70e5a0bdfa464decf120c137fe650688b6e079 Mon Sep 17 00:00:00 2001 From: dr0zd Date: Fri, 6 Feb 2026 06:50:41 +0700 Subject: [PATCH 11/11] finish pet-project clickhouse --- mdrozdov/clickhouse/stage2.sql | 143 +++++++++++++++++-------------- mdrozdov/clickhouse/stage3-4.sql | 45 ++++++++++ 2 files changed, 125 insertions(+), 63 deletions(-) create mode 100644 mdrozdov/clickhouse/stage3-4.sql diff --git a/mdrozdov/clickhouse/stage2.sql b/mdrozdov/clickhouse/stage2.sql index 1c74ed8..ec43feb 100644 --- a/mdrozdov/clickhouse/stage2.sql +++ b/mdrozdov/clickhouse/stage2.sql @@ -16,21 +16,27 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY (lesson_id, course_id); +-- здесь тригер таблица dim_lesson, хотя это не совсем правильно, но мы и состояние не храним +-- по сути таблица тригер должна быть fact_lesson_views, но right join будет плохо +-- а если не делать right join, то можем потерять логику, потому что left join от fact_lesson_view +-- логически теряет данные, если урок не был просмотрен +-- но оно хотя бы работает create materialized view mdrozdov.lesson_popularity_summary +on CLUSTER clickhouse_cluster to mdrozdov.dm_lesson_popularity_summary as select - dl.lesson_id as lesson_id, - dl.title as lesson_title, - dl.course_id as course_id, - dc.title as course_title, - count(flv.lesson_id) as total_views, - count(distinct flv.user_id) as unique_users, - min(flv.viewed_at) as first_view, - max(flv.viewed_at) as last_view -from mdrozdov.core_fact_lesson_views flv -right join mdrozdov.core_dim_course dc - on flv.course_id = dc.course_id -left join mdrozdov.core_dim_lesson dl + dl.lesson_id as lesson_id, + dl.title as lesson_title, + dl.course_id as course_id, + dc.title as course_title, + count(flv.lesson_id) as total_views, + count(distinct flv.user_id) as unique_users, + min(flv.viewed_at) as first_view, + max(flv.viewed_at) as last_view +from mdrozdov.core_dim_lesson dl +left join mdrozdov.core_dim_course dc + on dl.course_id = dc.course_id +left join mdrozdov.core_fact_lesson_views flv on dl.lesson_id = flv.lesson_id group by dl.lesson_id, dl.title, dl.course_id, dc.title @@ -50,23 +56,26 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY user_id +-- здесь тригер таблица dim_user +-- но почему-то при вставке в dim_user новые строки в таргет таблицу не добавляются create materialized view mdrozdov.inactive_users_summary - to mdrozdov.dm_inactive_users_summary - as - select - du.user_id as user_id, - du.name as name, - du.email as email, - du.age as age, - count(e.course_id) as registered_courses_count, - du.registration_date as registration_date - from mdrozdov.core_fact_lesson_views flv - right join mdrozdov.core_dim_user du - on du.user_id = flv.user_id - left join mdrozdov.core_enrollments e - on du.user_id = e.user_id - where flv.user_id is null - group by du.user_id, du.name, du.email, du.age, du.registration_date +on CLUSTER clickhouse_cluster +to mdrozdov.dm_inactive_users_summary +as +select + du.user_id as user_id, + du.name as name, + du.email as email, + du.age as age, + count(e.course_id) as registered_courses_count, + du.registration_date as registration_date +from mdrozdov.core_dim_user du +left join mdrozdov.core_enrollments e + on du.user_id = e.user_id +left join mdrozdov.core_fact_lesson_views flv + on du.user_id = flv.user_id +where flv.user_id is null +group by du.user_id, du.name, du.email, du.age, du.registration_date -- course_completion_rate create table mdrozdov.dm_course_completion_rate @@ -85,42 +94,44 @@ ENGINE = ReplicatedMergeTree( ) ORDER BY (user_id, course_id) +-- здесь тригер таблица core_enrollments и тут все четко по логике и в работе create materialized view mdrozdov.course_completion_rate - to mdrozdov.dm_course_completion_rate as - with total_lessons_per_course as( - select - course_id, - count(*) as lessons_in_course - from mdrozdov.core_dim_lesson - group by course_id - ), lessons_viewed as ( - select - user_id, - course_id, - count(*) as lessons_viewed - from mdrozdov.core_fact_lesson_views flv - group by user_id, course_id - ) +on CLUSTER clickhouse_cluster +to mdrozdov.dm_course_completion_rate as +with total_lessons_per_course as( + select + course_id, + count(*) as lessons_in_course + from mdrozdov.core_dim_lesson + group by course_id +), lessons_viewed as ( select - du.user_id as user_id, - du.name as user_name, - dc.course_id as course_id, - dc.title as course_title, - tlpc.lessons_in_course as lessons_in_course, - coalesce(lv.lessons_viewed, 0) as lessons_viewed, - round( - coalesce( - coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 - ), 2) as completion_rate - from mdrozdov.core_enrollments e - join mdrozdov.core_dim_user du - on e.user_id = du.user_id - join total_lessons_per_course tlpc - on tlpc.course_id = e.course_id - join mdrozdov.core_dim_course dc - on e.course_id = dc.course_id - left join lessons_viewed lv - on du.user_id = lv.user_id and dc.course_id = lv.course_id + user_id, + course_id, + count(*) as lessons_viewed + from mdrozdov.core_fact_lesson_views flv + group by user_id, course_id +) +select + du.user_id as user_id, + du.name as user_name, + dc.course_id as course_id, + dc.title as course_title, + tlpc.lessons_in_course as lessons_in_course, + coalesce(lv.lessons_viewed, 0) as lessons_viewed, + round( + coalesce( + coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 + ), 2) as completion_rate +from mdrozdov.core_enrollments e +join mdrozdov.core_dim_user du + on e.user_id = du.user_id +join total_lessons_per_course tlpc + on tlpc.course_id = e.course_id +join mdrozdov.core_dim_course dc + on e.course_id = dc.course_id +left join lessons_viewed lv + on du.user_id = lv.user_id and dc.course_id = lv.course_id --данные из-за несовпадения названий столбцов в импорте/экспорте пришлось удалять, короче проще вручную -- их добавить @@ -132,4 +143,10 @@ insert into mdrozdov.core_fact_lesson_views(user_id, lesson_id, course_id, viewe (4, 4, 3, '2023-04-02 10:00:00'), (4, 5, 3, '2023-04-03 10:00:00') --- данные в mdrozdov.core_enrollments загружал через импорт \ No newline at end of file +insert into enrollments (user_id, course_id, enrolled_at) values +(1, 1, '2023-01-15'), +(1, 2, '2023-02-01'), +(2, 1, '2023-01-20'), +(3, 2, '2023-03-05'), +(4, 3, '2023-04-01') + diff --git a/mdrozdov/clickhouse/stage3-4.sql b/mdrozdov/clickhouse/stage3-4.sql new file mode 100644 index 0000000..dc80643 --- /dev/null +++ b/mdrozdov/clickhouse/stage3-4.sql @@ -0,0 +1,45 @@ +-- Топ-5 уроков с наименьшим количеством уникальных пользователей +SELECT + lesson_title, + course_title, + unique_users +FROM lesson_popularity_summary +ORDER BY unique_users ASC +LIMIT 5; + +-- Пользователи без зарегистрированных курсов +SELECT + name, + email, + registration_date +FROM inactive_users_summary +WHERE registered_courses_count = 0 +ORDER BY registration_date ASC; + +-- Список курсов с минимальной средней степенью завершения +SELECT + course_title, + AVG(completion_rate) AS avg_completion +FROM course_completion_rate +GROUP BY course_title +ORDER BY avg_completion ASC +LIMIT 5; + +-- Мини отчет: +/* +Таблица lesson_popularity_summary + Выбранные поля для проверки: + total_views - агрегат count + first_view - агрегат min + оба совпадают +Таблица inactive_users_summary + Выбранные поля для проверки: + registered_courses_count - агрегат count + совпадает +Таблица course_completion_rate + Выбранные поля для проверки: + completion_rate - агрегат count + совпадает +Итог: в моем случае все значения совпали, возможно если бы это было не через простой MergeTree +все было бы хуже +*/ \ No newline at end of file