diff --git a/mdrozdov/clickhouse/array_map_nested.sql b/mdrozdov/clickhouse/array_map_nested.sql new file mode 100644 index 0000000..02959e9 --- /dev/null +++ b/mdrozdov/clickhouse/array_map_nested.sql @@ -0,0 +1,23 @@ +--task 1 + select + user_id, + indexOf(tags, 'promo') as promo_index, + arrayJoin(tags) + from events + where has(tags, 'promo') + + -- task 2 + select + user_id, + scores['math'] as math_score, + mapValues(scores) as all_values + from metrics + + -- task 3 + select + name, + contact.type as contact_type, + contact.value as contact_value + from users_nested + array join contacts as contact + where contact.type = 'email' \ No newline at end of file diff --git a/mdrozdov/clickhouse/data_types.sql b/mdrozdov/clickhouse/data_types.sql new file mode 100644 index 0000000..c7089fb --- /dev/null +++ b/mdrozdov/clickhouse/data_types.sql @@ -0,0 +1,61 @@ +-- task 1 + CREATE TABLE raw_data ( + user_id String, + balance String, + created_at String + ) ENGINE = MergeTree ORDER BY tuple(); + + select + user_id, + toDecimal64OrNull(balance, 2) as balance, + toTimeDate64OrNull(created_at, 3) as created_at + from raw_data + where + toDecimal64OrNull(balance, 2) is not null and + toTimeDate64OrNull(created_at, 3) is not null + + -- task 2 (failed) + CREATE TABLE events ( + event_id String, + event_date String, -- форматы: "2023-01-01", "01.02.2023", "invalid" + event_time String -- форматы: "12:30:45", "25:99:99" + ); + + select + event_id, + toStartOfHour(toDateTimeOrNull(concatWithSeparator(' ', events_date, events_time))) + from events + where + toDateOrNull(event_date) is not null and toTimeOrNull(event_time) is not null + + -- task 3 + CREATE TABLE user_activity ( + user_id UInt32, + actions String -- JSON: ["login", "purchase", "logout"] + ); + SELECT + user_id, + actions_arr, + length(actions_arr) AS actions_count + FROM + ( + SELECT + user_id, + JSONExtract(actions, 'Array(String)') AS actions_arr + FROM user_activity + ) + WHERE has(actions_arr, 'purchase'); + + -- task 4 + CREATE TABLE sensor_data ( + sensor_id UInt32, + value Float64, + status Enum('ok' = 1, 'error' = 2) + ); + SELECT + toString(status) as status, + avg(value) as avg_value + from sensor_data + where + toDecimal64OrNull(value, 2) is not null + group by status \ No newline at end of file diff --git a/mdrozdov/clickhouse/joins.sql b/mdrozdov/clickhouse/joins.sql new file mode 100644 index 0000000..9b797cb --- /dev/null +++ b/mdrozdov/clickhouse/joins.sql @@ -0,0 +1,103 @@ +--task 1 + create table mdrozdov.products ( + product_id Int, + name String + ) + engine = MergeTree + order by product_id + + insert into mdrozdov.products values + (101, 'iPhone 14'), + (102, 'Galaxy S22'), + (103, 'Pixel 7') + + create table mdrozdov.prices ( + product_id Int, + price Int, + region String + ) + engine = MergeTree + order by product_id + + insert into mdrozdov.prices values + (101, 999, 'US'), + (101, 1049, 'EU'), + (102, 850, 'US'), + (102, 899, 'CA'), + (103, 799, 'JP') + + -- 1) + select + product_id, + name, + price, + region + from mdrozdov.products + any left join mdrozdov.prices + on mdrozdov.products.product_id = mdrozdov.prices.product_id + + -- 2) + create table mdrozdov.ticks + ON CLUSTER clickhouse_cluster ( + time DateTime, + symbol String, + price Decimal(10, 2) + ) + engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/ticks', + '{replica}' + ) + order by (time, symbol) + + insert into mdrozdov.ticks values + ('2025-07-17 10:00:00', 'AAPL', 190.5), + ('2025-07-17 10:01:00', 'AAPL', 191.2), + ('2025-07-17 10:02:30', 'AAPL', 192.0), + ('2025-07-17 10:05:00', 'GOOG', 2850.0) + + create table mdrozdov.orders + ON CLUSTER clickhouse_cluster ( + time DateTime, + symbol String, + size Int + ) + engine = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/mdrozdov/orders', + '{replica}' + ) + order by size + + insert into mdrozdov.orders values + ('2025-07-17 10:00:30', 'AAPL', 50), + ('2025-07-17 10:02:00', 'AAPL', 75), + ('2025-07-17 10:03:00', 'AAPL', 20), + ('2025-07-17 10:04:00', 'GOOG', 15) + + select + o.time, + o.symbol, + t.price as last_price + from mdrozdov.orders o + asof join mdrozdov.ticks t + on o.symbol = t.symbol and o.time >= t.time + + -- task 3 + select + user_id, + name + from users + where user_id in ( + select user_id + from payments + where amount > 1000 + ) + + -- task 4 + select + bl.user_id, + u.name, + bl.event_time, + bl.event_type, + from big_logs bl + any left join users u + using(user_id) \ No newline at end of file diff --git a/mdrozdov/clickhouse/merge_tree_family.sql b/mdrozdov/clickhouse/merge_tree_family.sql new file mode 100644 index 0000000..71db981 --- /dev/null +++ b/mdrozdov/clickhouse/merge_tree_family.sql @@ -0,0 +1,64 @@ +--page_views table + create table page_views ( + user_id UInt64, + url String, + view_time DateTime + ) + Engine = MergeTree + Order by (user_id, view_time) + Partition by toYYYYMM(view_time) + + -- users table + create table users( + id UInt64, + name String, + updated_at DateTime + ) + ENGINE = ReplacingMergeTree(updated_at) + Order by id; + + -- region_sales table + create table region_sales( + region String, + date Date, + sales UInt64 + ) + ENGINE = SummingMergeTree + Partition by toYYYYMM(date) + Order by (region, date) + + insert into mdrozdov.region_sales (region, date, sales) + values ('Russia', '2025-01-14', 30), ('Russia', '2025-01-16', 22), ('USA', '2025-01-14', 1400) + OPTIMIZE TABLE mdrozdov.region_sales FINAL + + -- user_metrics table + create table mdrozdov.user_metrics ( + region String, + avg_age_state AggregateFunction(avg, UInt8), + active_users_state AggregateFunction(uniq, UInt64) + ) + Engine = AggregatingMergeTree + Order by region + + INSERT INTO mdrozdov.user_metrics + SELECT + 'Moscow' AS region, + avgState(toUInt8(25)) AS avg_age_state, + uniqState(toUInt64(100)) AS active_users_state + UNION ALL + SELECT + 'Moscow', + avgState(toUInt8(30)), + uniqState(toUInt64(50)) + UNION ALL + SELECT + 'SPB', + avgState(toUInt8(20)), + uniqState(toUInt64(10)); + + SELECT + region, + avgMerge(avg_age_state) AS avg_age, + uniqMerge(active_users_state) AS active_users + FROM mdrozdov.user_metrics + GROUP BY region \ No newline at end of file diff --git a/mdrozdov/clickhouse/queries.sql b/mdrozdov/clickhouse/queries.sql new file mode 100644 index 0000000..572ace7 --- /dev/null +++ b/mdrozdov/clickhouse/queries.sql @@ -0,0 +1,136 @@ +CREATE TABLE users ( + user_id Int, + name String, + age Int, + country String, + is_active bool + ) + engine = MergeTree + order by user_id + + insert into mdrozdov.users values + (1, 'Ivan' ,30, 'RU', 1), + (2, 'Anna' ,25, 'RU', 1), + (3, 'Petr' ,45, 'RU', 0), + (4, 'Olga' ,35, 'RU', 1), + (5, 'John' ,55, 'US', 1), + (6, 'Alice' ,42, 'US', 0), + (7, 'Tom' ,38, 'US', 1), + (8, 'Lucas' ,28, 'BR', 1), + (9, 'Maria' ,60, 'BR', 1), + (10, 'João' ,27, 'BR', 0), + (11, 'Kenji' ,33, 'JP', 1), + (12, 'Yuki' ,29, 'JP', 0), + (13, 'Chen' ,40, 'CN', 1), + (14, 'Li' ,48, 'CN', 1), + (15, 'Fatima',31, 'AE', 1), + (16, 'Ahmed' ,36, 'AE', 1), + (17, 'George',24, 'US', 1), + (18, 'Nina' ,34, 'RU', 1), + (19, 'Max' ,50, 'DE', 1), + (20, 'Sophie',22, 'DE', 0) + + --1. Получи 10 самых старших **активных** пользователей. + select + user_id, + name, + age, + country, + is_active + from mdrozdov.users + where is_active + order by age desc + limit 10 + --2. Выведи пользователей из `country = 'RU'`, отсортированных по имени. + select + user_id, + name, + age, + country, + is_active + from mdrozdov.users + where country = 'RU' + order by name asc + --3. Что произойдёт, если написать `LIMIT 10` **без `ORDER BY`**? + -- ответ будет недетерминирован + --4. Посчитай количество пользователей по странам. + select + country, + count(user_id) as total_users + from mdrozdov.users + group by country + --5. Для каждой страны выведи минимальный и максимальный возраст. + select + country, + min(age) as min_user_age, + max(age) as max_user_age + from mdrozdov.users + group by country + --6. Отсортируй страны по убыванию количества пользователей. + select + country, + count(user_id) as total_users + from mdrozdov.users + group by country + order by total_users desc + + -- task 2 + CREATE TABLE mdrozdov.events ( + user_id Int, + event_time DateTime, + event_type String --mb better enum but idc + ) + engine = MergeTree + order by user_id + insert into events values + (1, '2025-07-17 09:00:00', 'click'), + (1, '2025-07-17 09:05:00', 'view'), + (1, '2025-07-17 09:10:00', 'purchase'), + (2, '2025-07-17 10:00:00', 'click'), + (2, '2025-07-17 10:05:00', 'click'), + (2, '2025-07-17 11:00:00', 'view'), + (3, '2025-07-16 12:00:00', 'view'), + (3, '2025-07-16 12:30:00', 'click'), + (3, '2025-07-17 08:00:00', 'purchase'), + (4, '2025-07-17 13:00:00', 'click'), + (4, '2025-07-17 14:00:00', 'view'), + (4, '2025-07-17 15:00:00', 'view'), + (5, '2025-07-16 20:00:00', 'view'), + (5, '2025-07-17 01:00:00', 'click'), + (5, '2025-07-17 06:00:00', 'purchase'), + (6, '2025-07-17 07:30:00', 'view'), + (6, '2025-07-17 08:45:00', 'click'), + (7, '2025-07-16 23:59:59', 'click'), + (7, '2025-07-17 00:01:00', 'purchase'), + (7, '2025-07-17 10:10:00', 'view') + --1-2) Добавь колонку с кумулятивным количеством событий по user_id (используй count() OVER ...) + select + user_id, + event_time, + event_type, + count(*) over (partition by user_id order by event_time) as total_click_exact_moment, + row_number() over (partition by user_id order by event_time) as row_number + from mdrozdov.events + --3) да, поддерживается, но не обязательно для коммулятивного суммирования строк + --4-6) Сделай WITH recent_events AS (...) — все события за последние 24 часа. + with recent_events AS ( + select * + from mdrozdov.events + where event_time >= '2025-07-18'::DATE - interval '1 day' + ), category_list as ( + select 'click' as event_type, 'interaction' as category + union all + select 'view', 'interaction' + union all + select 'purchase', 'conversion' + ) + select + user_id, + count(*) as total_events_last_24h, + count(distinct cl.category) as uniq_categories + from recent_events re + left join category_list cl + using(event_type) + group by user_id + order by total_events_last_24h desc, user_id asc + limit 5 \ No newline at end of file diff --git a/mdrozdov/clickhouse/view_distribute.sql b/mdrozdov/clickhouse/view_distribute.sql new file mode 100644 index 0000000..d330749 --- /dev/null +++ b/mdrozdov/clickhouse/view_distribute.sql @@ -0,0 +1,33 @@ +-- task 1 + CREATE TABLE dist_table + ENGINE = Distributed( + 'cluster_name', -- нет доступа к конфигу infra clickhouse + 'mdrozdov', + 'logs', + rand() + ); + --rand() это параметр по которому мы распределяем данные при insert на разные ноды, в данном + --случае нам это нужно для равномерности распределения, потому что таблица не требует особых связей или тп + + -- task 2 + --Создай VIEW top_active_users, который показывает топ-100 пользователей по числу событий + CREATE VIEW top_active_users AS + select user_id, count(*) as total_events + from events + group by user_id + order by total_events desc + limit 100 + -- не подходит для дашборда с миллионами строк, потому что вычисляется каждый раз при вызове + -- для дашбордов лучше подойдет materialized view, тк хранит данные запроса и обновляется автоматически + -- при вставке новых данных + + --task 3 + CREATE MATERIALIZED VIEW + ENGINE = SummingMergeTree + order by date + mv_clicks_per_day AS + select + toDate(event_time) as date, + count(*) as total_clicks + from raw_events + group date \ No newline at end of file