diff --git a/mdrozdov/clickhouse/array_map_nested.sql b/mdrozdov/clickhouse/array_map_nested.sql new file mode 100644 index 0000000..02959e9 --- /dev/null +++ b/mdrozdov/clickhouse/array_map_nested.sql @@ -0,0 +1,23 @@ +--task 1 + select + user_id, + indexOf(tags, 'promo') as promo_index, + arrayJoin(tags) + from events + where has(tags, 'promo') + + -- task 2 + select + user_id, + scores['math'] as math_score, + mapValues(scores) as all_values + from metrics + + -- task 3 + select + name, + contact.type as contact_type, + contact.value as contact_value + from users_nested + array join contacts as contact + where contact.type = 'email' \ No newline at end of file diff --git a/mdrozdov/clickhouse/data_types.sql b/mdrozdov/clickhouse/data_types.sql new file mode 100644 index 0000000..fc580c3 --- /dev/null +++ b/mdrozdov/clickhouse/data_types.sql @@ -0,0 +1,61 @@ +-- task 1 + CREATE TABLE raw_data ( + user_id String, + balance String, + created_at String + ) ENGINE = MergeTree ORDER BY tuple(); + + select + user_id, + toDecimal64OrNull(balance, 2) as balance, + toDateTime64OrNull(created_at, 3) as created_at + from raw_data + where + toDecimal64OrNull(balance, 2) is not null and + toDateTime64OrNull(created_at, 3) is not null + + -- task 2 (failed) + CREATE TABLE events ( + event_id String, + event_date String, -- форматы: "2023-01-01", "01.02.2023", "invalid" + event_time String -- форматы: "12:30:45", "25:99:99" + ); + + select + event_id, + toStartOfHour(toDateTimeOrNull(concatWithSeparator(' ', event_date, event_time))) + from events + where + toDateOrNull(event_date) is not null and toTimeOrNull(event_time) is not null + + -- task 3 + CREATE TABLE user_activity ( + user_id UInt32, + actions String -- JSON: ["login", "purchase", "logout"] + ); + SELECT + user_id, + actions_arr, + length(actions_arr) AS actions_count + FROM + ( + SELECT + user_id, + JSONExtract(actions, 'Array(String)') AS actions_arr + FROM user_activity + ) + WHERE has(actions_arr, 'purchase'); + + -- task 4 + CREATE TABLE sensor_data ( + sensor_id UInt32, + value Float64, + status Enum('ok' = 1, 'error' = 2) + ); + SELECT + toString(status) as status, + avg(value) as avg_value + from sensor_data + where + value is not null + group by status \ No newline at end of file diff --git a/mdrozdov/clickhouse/joins.sql b/mdrozdov/clickhouse/joins.sql new file mode 100644 index 0000000..4701740 --- /dev/null +++ b/mdrozdov/clickhouse/joins.sql @@ -0,0 +1,103 @@ +--task 1 + create table mdrozdov.products ( + product_id Int, + name String + ) + engine = MergeTree + order by product_id + + insert into mdrozdov.products values + (101, 'iPhone 14'), + (102, 'Galaxy S22'), + (103, 'Pixel 7') + + create table mdrozdov.prices ( + product_id Int, + price Int, + region String + ) + engine = MergeTree + order by product_id + + insert into mdrozdov.prices values + (101, 999, 'US'), + (101, 1049, 'EU'), + (102, 850, 'US'), + (102, 899, 'CA'), + (103, 799, 'JP') + + -- 1) + select + product_id, + name, + price, + region + from mdrozdov.products + any left join mdrozdov.prices + on mdrozdov.products.product_id = mdrozdov.prices.product_id + + -- 2) + create table mdrozdov.ticks + ON CLUSTER clickhouse_cluster ( + time DateTime, + symbol String, + price Decimal(10, 2) + ) + engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/ticks', + '{replica}' + ) + order by (time, symbol) + + insert into mdrozdov.ticks values + ('2025-07-17 10:00:00', 'AAPL', 190.5), + ('2025-07-17 10:01:00', 'AAPL', 191.2), + ('2025-07-17 10:02:30', 'AAPL', 192.0), + ('2025-07-17 10:05:00', 'GOOG', 2850.0) + + create table mdrozdov.orders + ON CLUSTER clickhouse_cluster ( + time DateTime, + symbol String, + size Int + ) + engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/orders', + '{replica}' + ) + order by (symbol, time) + + insert into mdrozdov.orders values + ('2025-07-17 10:00:30', 'AAPL', 50), + ('2025-07-17 10:02:00', 'AAPL', 75), + ('2025-07-17 10:03:00', 'AAPL', 20), + ('2025-07-17 10:04:00', 'GOOG', 15) + + select + o.time, + o.symbol, + t.price as last_price + from mdrozdov.orders o + asof join mdrozdov.ticks t + on o.symbol = t.symbol and o.time >= t.time + + -- task 3 + select + user_id, + name + from users + where user_id in ( + select user_id + from payments + where amount > 1000 + ) + + -- task 4 + select + bl.user_id, + u.name, + bl.event_time, + bl.event_type + from big_logs bl + any left join users u + using(user_id) \ No newline at end of file diff --git a/mdrozdov/clickhouse/merge_tree_family.sql b/mdrozdov/clickhouse/merge_tree_family.sql new file mode 100644 index 0000000..74dbfba --- /dev/null +++ b/mdrozdov/clickhouse/merge_tree_family.sql @@ -0,0 +1,64 @@ +--page_views table + create table page_views ( + user_id UInt64, + url String, + view_time DateTime + ) + Engine = MergeTree + Order by (user_id, view_time) + Partition by toYYYYMM(view_time) + + -- users table + create table users( + id UInt64, + name String, + updated_at DateTime + ) + ENGINE = ReplacingMergeTree(updated_at) + Order by id; + + -- region_sales table + create table mdrozdov.region_sales( + region String, + date Date, + sales UInt64 + ) + ENGINE = SummingMergeTree + Partition by toYYYYMM(date) + Order by (region, date) + + insert into mdrozdov.region_sales (region, date, sales) + values ('Russia', '2025-01-14', 30), ('Russia', '2025-01-16', 22), ('USA', '2025-01-14', 1400) + OPTIMIZE TABLE mdrozdov.region_sales FINAL + + -- user_metrics table + create table mdrozdov.user_metrics ( + region String, + avg_age_state AggregateFunction(avg, UInt8), + active_users_state AggregateFunction(uniq, UInt64) + ) + Engine = AggregatingMergeTree + Order by region + + INSERT INTO mdrozdov.user_metrics + SELECT + 'Moscow' AS region, + avgState(toUInt8(25)) AS avg_age_state, + uniqState(toUInt64(100)) AS active_users_state + UNION ALL + SELECT + 'Moscow', + avgState(toUInt8(30)), + uniqState(toUInt64(50)) + UNION ALL + SELECT + 'SPB', + avgState(toUInt8(20)), + uniqState(toUInt64(10)); + + SELECT + region, + avgMerge(avg_age_state) AS avg_age, + uniqMerge(active_users_state) AS active_users + FROM mdrozdov.user_metrics + GROUP BY region \ No newline at end of file diff --git a/mdrozdov/clickhouse/partitions.sql b/mdrozdov/clickhouse/partitions.sql new file mode 100644 index 0000000..817d95b --- /dev/null +++ b/mdrozdov/clickhouse/partitions.sql @@ -0,0 +1,98 @@ +-- task 1 +create table mdrozdov.sales2 +on CLUSTER clickhouse_cluster ( + id Int, + region String, + sale_date Date, + amount Decimal(8, 2) +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/sales2', + '{replica}' + ) +partition by toYYYYMM(sale_date) +order by id +-- удаление таблицы +-- drop table mdrozdov.sales +-- on CLUSTER clickhouse_cluster; не удаляет таблицу полностью, она остается в zookeeper +INSERT into mdrozdov.sales2 values +(1, 'RU', '2024-05-01', 100.0), +(2, 'RU', '2024-05-15', 120.0), --эти значения удалились +(3, 'US', '2024-06-01', 90.0), +(4, 'US', '2024-07-03', 150.0), +(5, 'DE', '2024-07-04', 160.0) +alter table mdrozdov.sales2 +drop partition '202405' +--2 значения удалились, которые хранились в соответствующей партиции + +-- task 2 +create table mdrozdov.events2 +on CLUSTER clickhouse_cluster ( + id Int, + event_type String, + event_date Date +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/events2', + '{replica}' + ) +partition by event_type +order by id + +INSERT into mdrozdov.events2 values +(1, 'login', '2024-06-01'), +(2, 'click', '2024-06-01'), +(3, 'click', '2024-06-02'), +(4, 'purchase', '2024-06-03'), +(5, 'login', '2024-06-04') + +alter table mdrozdov.events2 +drop partition 'click' + +-- task 3 +create table mdrozdov.logs +on CLUSTER clickhouse_cluster ( + id Int, + user_id Int, + log_time DateTime +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/logs', + '{replica}' + ) +partition by cityHash64(user_id) % 100 +order by id + +insert into mdrozdov.logs values +(1, 123, '2024-07-17 10:00:00'), +(2, 456, '2024-07-17 10:01:00'), +(3, 789, '2024-07-17 10:02:00') +--Что ты выиграешь с точки зрения баланса? +--равномерное распределение данных в разных партициях, лучшее управление обновлением и удалением данных +-- Почему такой тип PARTITION не ускорит SELECT * WHERE user_id = 456? +-- потому что clickhouse не умеет выполнять условия партиции, чтобы найти сразу ту в которой надо искать, +-- поэтому будет пробегать все 100, чтобы найти нужное, поэтому важно, чтобы таблица была отсортирована по user_id + +-- task 4 +create table mdrozdov.visits +on CLUSTER clickhouse_cluster ( + id Int, + region String, + visit_date Date +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/visits', + '{replica}' + ) +partition by (region, toYYYYMM(visit_date)) +order by (region, visit_date) + +insert into mdrozdov.visits values +(1, 'RU', '2024-06-01'), +(2, 'RU', '2024-06-15'), +(3, 'US', '2024-06-03'), +(4, 'US', '2024-07-01'), +(5, 'RU', '2024-07-05') + +alter table mdrozdov.visits +drop partition ('RU', 202407) -- эффективное удаление июльских данных по россии \ No newline at end of file diff --git a/mdrozdov/clickhouse/queries.sql b/mdrozdov/clickhouse/queries.sql new file mode 100644 index 0000000..e2d7afe --- /dev/null +++ b/mdrozdov/clickhouse/queries.sql @@ -0,0 +1,136 @@ +CREATE TABLE mdrozdov.users ( + user_id Int, + name String, + age Int, + country String, + is_active bool + ) + engine = MergeTree + order by user_id + + insert into mdrozdov.users values + (1, 'Ivan' ,30, 'RU', 1), + (2, 'Anna' ,25, 'RU', 1), + (3, 'Petr' ,45, 'RU', 0), + (4, 'Olga' ,35, 'RU', 1), + (5, 'John' ,55, 'US', 1), + (6, 'Alice' ,42, 'US', 0), + (7, 'Tom' ,38, 'US', 1), + (8, 'Lucas' ,28, 'BR', 1), + (9, 'Maria' ,60, 'BR', 1), + (10, 'João' ,27, 'BR', 0), + (11, 'Kenji' ,33, 'JP', 1), + (12, 'Yuki' ,29, 'JP', 0), + (13, 'Chen' ,40, 'CN', 1), + (14, 'Li' ,48, 'CN', 1), + (15, 'Fatima',31, 'AE', 1), + (16, 'Ahmed' ,36, 'AE', 1), + (17, 'George',24, 'US', 1), + (18, 'Nina' ,34, 'RU', 1), + (19, 'Max' ,50, 'DE', 1), + (20, 'Sophie',22, 'DE', 0) + + --1. Получи 10 самых старших **активных** пользователей. + select + user_id, + name, + age, + country, + is_active + from mdrozdov.users + where is_active + order by age desc + limit 10 + --2. Выведи пользователей из `country = 'RU'`, отсортированных по имени. + select + user_id, + name, + age, + country, + is_active + from mdrozdov.users + where country = 'RU' + order by name asc + --3. Что произойдёт, если написать `LIMIT 10` **без `ORDER BY`**? + -- ответ будет недетерминирован + --4. Посчитай количество пользователей по странам. + select + country, + count(user_id) as total_users + from mdrozdov.users + group by country + --5. Для каждой страны выведи минимальный и максимальный возраст. + select + country, + min(age) as min_user_age, + max(age) as max_user_age + from mdrozdov.users + group by country + --6. Отсортируй страны по убыванию количества пользователей. + select + country, + count(user_id) as total_users + from mdrozdov.users + group by country + order by total_users desc + + -- task 2 + CREATE TABLE mdrozdov.events ( + user_id Int, + event_time DateTime, + event_type String --mb better enum but idc + ) + engine = MergeTree + order by user_id + insert into events values + (1, '2025-07-17 09:00:00', 'click'), + (1, '2025-07-17 09:05:00', 'view'), + (1, '2025-07-17 09:10:00', 'purchase'), + (2, '2025-07-17 10:00:00', 'click'), + (2, '2025-07-17 10:05:00', 'click'), + (2, '2025-07-17 11:00:00', 'view'), + (3, '2025-07-16 12:00:00', 'view'), + (3, '2025-07-16 12:30:00', 'click'), + (3, '2025-07-17 08:00:00', 'purchase'), + (4, '2025-07-17 13:00:00', 'click'), + (4, '2025-07-17 14:00:00', 'view'), + (4, '2025-07-17 15:00:00', 'view'), + (5, '2025-07-16 20:00:00', 'view'), + (5, '2025-07-17 01:00:00', 'click'), + (5, '2025-07-17 06:00:00', 'purchase'), + (6, '2025-07-17 07:30:00', 'view'), + (6, '2025-07-17 08:45:00', 'click'), + (7, '2025-07-16 23:59:59', 'click'), + (7, '2025-07-17 00:01:00', 'purchase'), + (7, '2025-07-17 10:10:00', 'view') + --1-2) Добавь колонку с кумулятивным количеством событий по user_id (используй count() OVER ...) + select + user_id, + event_time, + event_type, + count(*) over (partition by user_id order by event_time) as total_click_exact_moment, + row_number() over (partition by user_id order by event_time) as row_number + from mdrozdov.events + --3) да, поддерживается, но не обязательно для коммулятивного суммирования строк + --4-6) Сделай WITH recent_events AS (...) — все события за последние 24 часа. + with mdrozdov.recent_events AS ( + select * + from mdrozdov.events + where event_time >= toDateTime('2025-07-18') - interval 1 day + ), category_list as ( + select 'click' as event_type, 'interaction' as category + union all + select 'view', 'interaction' + union all + select 'purchase', 'conversion' + ) + select + user_id, + count(*) as total_events_last_24h, + count(distinct cl.category) as uniq_categories + from recent_events re + left join category_list cl + using(event_type) + group by user_id + order by total_events_last_24h desc, user_id asc + limit 5 \ No newline at end of file diff --git a/mdrozdov/clickhouse/secondary_index.sql b/mdrozdov/clickhouse/secondary_index.sql new file mode 100644 index 0000000..53f172c --- /dev/null +++ b/mdrozdov/clickhouse/secondary_index.sql @@ -0,0 +1,65 @@ +--task 1 +create table mdrozdov.logs3 +on CLUSTER clickhouse_cluster ( + log_id Int, + ts DateTime +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/logs3', + '{replica}' + ) +order by ts + +alter table mdrozdov.logs3 +add index idx_ts ts type minmax GRANULARITY 2; + +select * +from mdrozdov.logs3 +where ts between '2024-06-02' and '2024-06-03' + +SELECT + query, + read_rows, + result_rows, + read_bytes +FROM system.query_log +WHERE query LIKE '%logs%' +ORDER BY event_time DESC +LIMIT 1; + +-- task 2 +create table mdrozdov.clicks +on CLUSTER clickhouse_cluster ( + id UInt64, + country String +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/clicks', + '{replica}' + ) +order by id + +alter table mdrozdov.clicks +add index idx_country country type set(100) GRANULARITY 2 + +Explain + select * + from mdrozdov.clicks + where country = 'RU' + +-- task 3 +create table mdrozdov.users +on CLUSTER clickhouse_cluster ( + id UInt64, + email String +) +engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/users', + '{replica}' + ) +order by id + +ALTER TABLE mdrozdov.users +ADD INDEX idx_email email TYPE bloom_filter(0.01) GRANULARITY 2; + +SELECT * FROM mdrozdov.users WHERE email = 'alice@mail.com'; \ No newline at end of file diff --git a/mdrozdov/clickhouse/view_distribute.sql b/mdrozdov/clickhouse/view_distribute.sql new file mode 100644 index 0000000..41ed079 --- /dev/null +++ b/mdrozdov/clickhouse/view_distribute.sql @@ -0,0 +1,34 @@ +-- task 1 + CREATE TABLE dist_table + ENGINE = Distributed( + 'cluster_name', -- нет доступа к конфигу infra clickhouse + 'mdrozdov', + 'logs', + rand() + ); + --rand() это параметр по которому мы распределяем данные при insert на разные ноды, в данном + --случае нам это нужно для равномерности распределения, потому что таблица не требует особых связей или тп + + -- task 2 + --Создай VIEW top_active_users, который показывает топ-100 пользователей по числу событий + CREATE VIEW top_active_users AS + select user_id, count(*) as total_events + from events + group by user_id + order by total_events desc + limit 100 + -- не подходит для дашборда с миллионами строк, потому что вычисляется каждый раз при вызове + -- для дашбордов лучше подойдет materialized view, тк хранит данные запроса и обновляется автоматически + -- при вставке новых данных + + --task 3 + CREATE MATERIALIZED VIEW + to + ENGINE = SummingMergeTree + order by date + mv_clicks_per_day AS + select + toDate(event_time) as date, + count(*) as total_clicks + from raw_events + group by date \ No newline at end of file diff --git a/mdrozdov/greenplum/core_layer.sql b/mdrozdov/greenplum/core_layer.sql new file mode 100644 index 0000000..8ff3a53 --- /dev/null +++ b/mdrozdov/greenplum/core_layer.sql @@ -0,0 +1,94 @@ +-- Измерение пользователей + CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE + ) + distributed by (user_id); + + -- Измерение курсов + CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT + ) + distributed by (course_id); + + -- Измерение уроков + CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) + ) + distributed by (lesson_id); + + -- Опционально: измерение времени + CREATE TABLE core.dim_date ( + date DATE PRIMARY KEY, + year INT, + month INT, + day INT, + weekday TEXT + ) + distributed replicated; + + -- Таблицы фактов: + CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP + ) + distributed by (course_id); + + CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP + ) + distributed by (course_id); + + --ETL переброс данных из raw в core + INSERT INTO core.dim_user (user_id, name, age, email, registration_date) + SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE + FROM raw.users; + + INSERT INTO core.dim_course (course_id, title, category) + SELECT DISTINCT + id::INT, + title::TEXT, + category::TEXT + FROM raw.courses; + + INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) + SELECT DISTINCT + id::INT, + title::TEXT, + duration_min::INT, + course_id::INT + FROM raw.lessons; + + -- Заполнение таблицы фактов + insert into core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) + select + lw.user_id, + lw.lesson_id, + l.course_id, + lw.viewed_at + from raw.lessons_views lw + join raw.lessons l on lw.lesson_id = l.id; + + insert into core.enrollments (user_id, course_id, enrolled_at) + select + user_id, + course_id, + enrolled_at + from raw.enrollments; \ No newline at end of file diff --git a/mdrozdov/greenplum/dm_layer.sql b/mdrozdov/greenplum/dm_layer.sql new file mode 100644 index 0000000..991db90 --- /dev/null +++ b/mdrozdov/greenplum/dm_layer.sql @@ -0,0 +1,71 @@ +--lesson_popularity_summary data mart + create materialized view dm.lesson_popularity_summary as + select + dl.lesson_id, + dl.title as lesson_title, + dl.course_id, + dc.title as course_title, + count(flv.lesson_id) as total_views, + count(distinct flv.user_id) as unique_users, + min(flv.viewed_at) as first_view, + max(flv.viewed_at) as last_view + from core.dim_lesson dl + left join core.dim_course dc + on dl.course_id = dc.course_id + left join core.fact_lesson_views flv + on dl.lesson_id = flv.lesson_id + group by dl.lesson_id, dl.title, dl.course_id, dc.title + + -- inactive_users_summary data mart + create materialized view dm.inactive_users_summary as + select + du.user_id, + du.name as user_name, + du.email, + du.age, + count(e.course_id) as enrollments_count, + du.registration_date + from core.dim_user du + left join core.enrollments e + on du.user_id = e.user_id + left join core.fact_lesson_views flv + on du.user_id = flv.user_id + where flv.user_id is null + group by du.user_id, du.name, du.email, du.age, du.registration_date + + -- course_completion_rate data mart + create materialized view dm.course_completion_rate as + with total_lessons_per_course as( + select + course_id, + count(*) as lessons_in_course + from core.dim_lesson + group by course_id + ), lessons_viewed as ( + select + user_id, + course_id, + count(*) as lessons_viewed + from fact_lesson_views flv + group by user_id, course_id + ) + select + du.user_id as user_id, + du.name as user_name, + dc.course_id as course_id, + dc.title as course_title, + tlpc.lessons_in_course, + coalesce(lv.lessons_viewed, 0) as lessons_viewed, + round( + coalesce( + coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0 + ), 2) as completion_rate + from core.enrollments e + join core.dim_user du + on e.user_id = du.user_id + join total_lessons_per_course tlpc + on tlpc.course_id = e.course_id + join core.dim_course dc + on e.course_id = dc.course_id + left join lessons_viewed lv + on du.user_id = lv.user_id and dc.course_id = lv.course_id \ No newline at end of file diff --git a/mdrozdov/greenplum/raw_layer.sql b/mdrozdov/greenplum/raw_layer.sql new file mode 100644 index 0000000..d2ab8c9 --- /dev/null +++ b/mdrozdov/greenplum/raw_layer.sql @@ -0,0 +1,44 @@ +create schema raw; + create schema core; + create schema dm; + + create table raw.users( + id INT, + name varchar(50), + age INT, + email varchar(50), + registration_date date + ) + distributed by (id); + + create table raw.courses ( + id INT, + title text, + category varchar(40), + created_at date + ) + distributed by (id); + + create table raw.lessons ( + id INT, + course_id INT, + title TEXT, + duration_min INT + ) + distributed by (course_id); + + create table raw.enrollments ( + id INT, + user_id INT, + course_id INT, + enrolled_at date + ) + distributed by (user_id); + + create table raw.lessons_views( + id int, + user_id int, + lesson_id int, + viewed_at timestamp + ) + distributed by (user_id) \ No newline at end of file