Merge pull request 'OS-401' (#69) from OS-401 into main
Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/69 Reviewed-by: aleksey bogatyrev <abogatyrev@example.com>
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
--task 1
|
||||
select
|
||||
user_id,
|
||||
indexOf(tags, 'promo') as promo_index,
|
||||
arrayJoin(tags)
|
||||
from events
|
||||
where has(tags, 'promo')
|
||||
|
||||
-- task 2
|
||||
select
|
||||
user_id,
|
||||
scores['math'] as math_score,
|
||||
mapValues(scores) as all_values
|
||||
from metrics
|
||||
|
||||
-- task 3
|
||||
select
|
||||
name,
|
||||
contact.type as contact_type,
|
||||
contact.value as contact_value
|
||||
from users_nested
|
||||
array join contacts as contact
|
||||
where contact.type = 'email'
|
||||
@@ -0,0 +1,61 @@
|
||||
-- task 1
|
||||
CREATE TABLE raw_data (
|
||||
user_id String,
|
||||
balance String,
|
||||
created_at String
|
||||
) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
select
|
||||
user_id,
|
||||
toDecimal64OrNull(balance, 2) as balance,
|
||||
toDateTime64OrNull(created_at, 3) as created_at
|
||||
from raw_data
|
||||
where
|
||||
toDecimal64OrNull(balance, 2) is not null and
|
||||
toDateTime64OrNull(created_at, 3) is not null
|
||||
|
||||
-- task 2 (failed)
|
||||
CREATE TABLE events (
|
||||
event_id String,
|
||||
event_date String, -- форматы: "2023-01-01", "01.02.2023", "invalid"
|
||||
event_time String -- форматы: "12:30:45", "25:99:99"
|
||||
);
|
||||
|
||||
select
|
||||
event_id,
|
||||
toStartOfHour(toDateTimeOrNull(concatWithSeparator(' ', event_date, event_time)))
|
||||
from events
|
||||
where
|
||||
toDateOrNull(event_date) is not null and toTimeOrNull(event_time) is not null
|
||||
|
||||
-- task 3
|
||||
CREATE TABLE user_activity (
|
||||
user_id UInt32,
|
||||
actions String -- JSON: ["login", "purchase", "logout"]
|
||||
);
|
||||
SELECT
|
||||
user_id,
|
||||
actions_arr,
|
||||
length(actions_arr) AS actions_count
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
user_id,
|
||||
JSONExtract(actions, 'Array(String)') AS actions_arr
|
||||
FROM user_activity
|
||||
)
|
||||
WHERE has(actions_arr, 'purchase');
|
||||
|
||||
-- task 4
|
||||
CREATE TABLE sensor_data (
|
||||
sensor_id UInt32,
|
||||
value Float64,
|
||||
status Enum('ok' = 1, 'error' = 2)
|
||||
);
|
||||
SELECT
|
||||
toString(status) as status,
|
||||
avg(value) as avg_value
|
||||
from sensor_data
|
||||
where
|
||||
value is not null
|
||||
group by status
|
||||
@@ -0,0 +1,103 @@
|
||||
--task 1
|
||||
create table mdrozdov.products (
|
||||
product_id Int,
|
||||
name String
|
||||
)
|
||||
engine = MergeTree
|
||||
order by product_id
|
||||
|
||||
insert into mdrozdov.products values
|
||||
(101, 'iPhone 14'),
|
||||
(102, 'Galaxy S22'),
|
||||
(103, 'Pixel 7')
|
||||
|
||||
create table mdrozdov.prices (
|
||||
product_id Int,
|
||||
price Int,
|
||||
region String
|
||||
)
|
||||
engine = MergeTree
|
||||
order by product_id
|
||||
|
||||
insert into mdrozdov.prices values
|
||||
(101, 999, 'US'),
|
||||
(101, 1049, 'EU'),
|
||||
(102, 850, 'US'),
|
||||
(102, 899, 'CA'),
|
||||
(103, 799, 'JP')
|
||||
|
||||
-- 1)
|
||||
select
|
||||
product_id,
|
||||
name,
|
||||
price,
|
||||
region
|
||||
from mdrozdov.products
|
||||
any left join mdrozdov.prices
|
||||
on mdrozdov.products.product_id = mdrozdov.prices.product_id
|
||||
|
||||
-- 2)
|
||||
create table mdrozdov.ticks
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
time DateTime,
|
||||
symbol String,
|
||||
price Decimal(10, 2)
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/ticks',
|
||||
'{replica}'
|
||||
)
|
||||
order by (time, symbol)
|
||||
|
||||
insert into mdrozdov.ticks values
|
||||
('2025-07-17 10:00:00', 'AAPL', 190.5),
|
||||
('2025-07-17 10:01:00', 'AAPL', 191.2),
|
||||
('2025-07-17 10:02:30', 'AAPL', 192.0),
|
||||
('2025-07-17 10:05:00', 'GOOG', 2850.0)
|
||||
|
||||
create table mdrozdov.orders
|
||||
ON CLUSTER clickhouse_cluster (
|
||||
time DateTime,
|
||||
symbol String,
|
||||
size Int
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/orders',
|
||||
'{replica}'
|
||||
)
|
||||
order by (symbol, time)
|
||||
|
||||
insert into mdrozdov.orders values
|
||||
('2025-07-17 10:00:30', 'AAPL', 50),
|
||||
('2025-07-17 10:02:00', 'AAPL', 75),
|
||||
('2025-07-17 10:03:00', 'AAPL', 20),
|
||||
('2025-07-17 10:04:00', 'GOOG', 15)
|
||||
|
||||
select
|
||||
o.time,
|
||||
o.symbol,
|
||||
t.price as last_price
|
||||
from mdrozdov.orders o
|
||||
asof join mdrozdov.ticks t
|
||||
on o.symbol = t.symbol and o.time >= t.time
|
||||
|
||||
-- task 3
|
||||
select
|
||||
user_id,
|
||||
name
|
||||
from users
|
||||
where user_id in (
|
||||
select user_id
|
||||
from payments
|
||||
where amount > 1000
|
||||
)
|
||||
|
||||
-- task 4
|
||||
select
|
||||
bl.user_id,
|
||||
u.name,
|
||||
bl.event_time,
|
||||
bl.event_type
|
||||
from big_logs bl
|
||||
any left join users u
|
||||
using(user_id)
|
||||
@@ -0,0 +1,64 @@
|
||||
--page_views table
|
||||
create table page_views (
|
||||
user_id UInt64,
|
||||
url String,
|
||||
view_time DateTime
|
||||
)
|
||||
Engine = MergeTree
|
||||
Order by (user_id, view_time)
|
||||
Partition by toYYYYMM(view_time)
|
||||
|
||||
-- users table
|
||||
create table users(
|
||||
id UInt64,
|
||||
name String,
|
||||
updated_at DateTime
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(updated_at)
|
||||
Order by id;
|
||||
|
||||
-- region_sales table
|
||||
create table mdrozdov.region_sales(
|
||||
region String,
|
||||
date Date,
|
||||
sales UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree
|
||||
Partition by toYYYYMM(date)
|
||||
Order by (region, date)
|
||||
|
||||
insert into mdrozdov.region_sales (region, date, sales)
|
||||
values ('Russia', '2025-01-14', 30), ('Russia', '2025-01-16', 22), ('USA', '2025-01-14', 1400)
|
||||
OPTIMIZE TABLE mdrozdov.region_sales FINAL
|
||||
|
||||
-- user_metrics table
|
||||
create table mdrozdov.user_metrics (
|
||||
region String,
|
||||
avg_age_state AggregateFunction(avg, UInt8),
|
||||
active_users_state AggregateFunction(uniq, UInt64)
|
||||
)
|
||||
Engine = AggregatingMergeTree
|
||||
Order by region
|
||||
|
||||
INSERT INTO mdrozdov.user_metrics
|
||||
SELECT
|
||||
'Moscow' AS region,
|
||||
avgState(toUInt8(25)) AS avg_age_state,
|
||||
uniqState(toUInt64(100)) AS active_users_state
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Moscow',
|
||||
avgState(toUInt8(30)),
|
||||
uniqState(toUInt64(50))
|
||||
UNION ALL
|
||||
SELECT
|
||||
'SPB',
|
||||
avgState(toUInt8(20)),
|
||||
uniqState(toUInt64(10));
|
||||
|
||||
SELECT
|
||||
region,
|
||||
avgMerge(avg_age_state) AS avg_age,
|
||||
uniqMerge(active_users_state) AS active_users
|
||||
FROM mdrozdov.user_metrics
|
||||
GROUP BY region
|
||||
@@ -0,0 +1,98 @@
|
||||
-- task 1
|
||||
create table mdrozdov.sales2
|
||||
on CLUSTER clickhouse_cluster (
|
||||
id Int,
|
||||
region String,
|
||||
sale_date Date,
|
||||
amount Decimal(8, 2)
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/sales2',
|
||||
'{replica}'
|
||||
)
|
||||
partition by toYYYYMM(sale_date)
|
||||
order by id
|
||||
-- удаление таблицы
|
||||
-- drop table mdrozdov.sales
|
||||
-- on CLUSTER clickhouse_cluster; не удаляет таблицу полностью, она остается в zookeeper
|
||||
INSERT into mdrozdov.sales2 values
|
||||
(1, 'RU', '2024-05-01', 100.0),
|
||||
(2, 'RU', '2024-05-15', 120.0), --эти значения удалились
|
||||
(3, 'US', '2024-06-01', 90.0),
|
||||
(4, 'US', '2024-07-03', 150.0),
|
||||
(5, 'DE', '2024-07-04', 160.0)
|
||||
alter table mdrozdov.sales2
|
||||
drop partition '202405'
|
||||
--2 значения удалились, которые хранились в соответствующей партиции
|
||||
|
||||
-- task 2
|
||||
create table mdrozdov.events2
|
||||
on CLUSTER clickhouse_cluster (
|
||||
id Int,
|
||||
event_type String,
|
||||
event_date Date
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/events2',
|
||||
'{replica}'
|
||||
)
|
||||
partition by event_type
|
||||
order by id
|
||||
|
||||
INSERT into mdrozdov.events2 values
|
||||
(1, 'login', '2024-06-01'),
|
||||
(2, 'click', '2024-06-01'),
|
||||
(3, 'click', '2024-06-02'),
|
||||
(4, 'purchase', '2024-06-03'),
|
||||
(5, 'login', '2024-06-04')
|
||||
|
||||
alter table mdrozdov.events2
|
||||
drop partition 'click'
|
||||
|
||||
-- task 3
|
||||
create table mdrozdov.logs
|
||||
on CLUSTER clickhouse_cluster (
|
||||
id Int,
|
||||
user_id Int,
|
||||
log_time DateTime
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/logs',
|
||||
'{replica}'
|
||||
)
|
||||
partition by cityHash64(user_id) % 100
|
||||
order by id
|
||||
|
||||
insert into mdrozdov.logs values
|
||||
(1, 123, '2024-07-17 10:00:00'),
|
||||
(2, 456, '2024-07-17 10:01:00'),
|
||||
(3, 789, '2024-07-17 10:02:00')
|
||||
--Что ты выиграешь с точки зрения баланса?
|
||||
--равномерное распределение данных в разных партициях, лучшее управление обновлением и удалением данных
|
||||
-- Почему такой тип PARTITION не ускорит SELECT * WHERE user_id = 456?
|
||||
-- потому что clickhouse не умеет выполнять условия партиции, чтобы найти сразу ту в которой надо искать,
|
||||
-- поэтому будет пробегать все 100, чтобы найти нужное, поэтому важно, чтобы таблица была отсортирована по user_id
|
||||
|
||||
-- task 4
|
||||
create table mdrozdov.visits
|
||||
on CLUSTER clickhouse_cluster (
|
||||
id Int,
|
||||
region String,
|
||||
visit_date Date
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/visits',
|
||||
'{replica}'
|
||||
)
|
||||
partition by (region, toYYYYMM(visit_date))
|
||||
order by (region, visit_date)
|
||||
|
||||
insert into mdrozdov.visits values
|
||||
(1, 'RU', '2024-06-01'),
|
||||
(2, 'RU', '2024-06-15'),
|
||||
(3, 'US', '2024-06-03'),
|
||||
(4, 'US', '2024-07-01'),
|
||||
(5, 'RU', '2024-07-05')
|
||||
|
||||
alter table mdrozdov.visits
|
||||
drop partition ('RU', 202407) -- эффективное удаление июльских данных по россии
|
||||
@@ -0,0 +1,136 @@
|
||||
CREATE TABLE mdrozdov.users (
|
||||
user_id Int,
|
||||
name String,
|
||||
age Int,
|
||||
country String,
|
||||
is_active bool
|
||||
)
|
||||
engine = MergeTree
|
||||
order by user_id
|
||||
|
||||
insert into mdrozdov.users values
|
||||
(1, 'Ivan' ,30, 'RU', 1),
|
||||
(2, 'Anna' ,25, 'RU', 1),
|
||||
(3, 'Petr' ,45, 'RU', 0),
|
||||
(4, 'Olga' ,35, 'RU', 1),
|
||||
(5, 'John' ,55, 'US', 1),
|
||||
(6, 'Alice' ,42, 'US', 0),
|
||||
(7, 'Tom' ,38, 'US', 1),
|
||||
(8, 'Lucas' ,28, 'BR', 1),
|
||||
(9, 'Maria' ,60, 'BR', 1),
|
||||
(10, 'João' ,27, 'BR', 0),
|
||||
(11, 'Kenji' ,33, 'JP', 1),
|
||||
(12, 'Yuki' ,29, 'JP', 0),
|
||||
(13, 'Chen' ,40, 'CN', 1),
|
||||
(14, 'Li' ,48, 'CN', 1),
|
||||
(15, 'Fatima',31, 'AE', 1),
|
||||
(16, 'Ahmed' ,36, 'AE', 1),
|
||||
(17, 'George',24, 'US', 1),
|
||||
(18, 'Nina' ,34, 'RU', 1),
|
||||
(19, 'Max' ,50, 'DE', 1),
|
||||
(20, 'Sophie',22, 'DE', 0)
|
||||
|
||||
--1. Получи 10 самых старших **активных** пользователей.
|
||||
select
|
||||
user_id,
|
||||
name,
|
||||
age,
|
||||
country,
|
||||
is_active
|
||||
from mdrozdov.users
|
||||
where is_active
|
||||
order by age desc
|
||||
limit 10
|
||||
--2. Выведи пользователей из `country = 'RU'`, отсортированных по имени.
|
||||
select
|
||||
user_id,
|
||||
name,
|
||||
age,
|
||||
country,
|
||||
is_active
|
||||
from mdrozdov.users
|
||||
where country = 'RU'
|
||||
order by name asc
|
||||
--3. Что произойдёт, если написать `LIMIT 10` **без `ORDER BY`**?
|
||||
-- ответ будет недетерминирован
|
||||
--4. Посчитай количество пользователей по странам.
|
||||
select
|
||||
country,
|
||||
count(user_id) as total_users
|
||||
from mdrozdov.users
|
||||
group by country
|
||||
--5. Для каждой страны выведи минимальный и максимальный возраст.
|
||||
select
|
||||
country,
|
||||
min(age) as min_user_age,
|
||||
max(age) as max_user_age
|
||||
from mdrozdov.users
|
||||
group by country
|
||||
--6. Отсортируй страны по убыванию количества пользователей.
|
||||
select
|
||||
country,
|
||||
count(user_id) as total_users
|
||||
from mdrozdov.users
|
||||
group by country
|
||||
order by total_users desc
|
||||
|
||||
-- task 2
|
||||
CREATE TABLE mdrozdov.events (
|
||||
user_id Int,
|
||||
event_time DateTime,
|
||||
event_type String --mb better enum but idc
|
||||
)
|
||||
engine = MergeTree
|
||||
order by user_id
|
||||
insert into events values
|
||||
(1, '2025-07-17 09:00:00', 'click'),
|
||||
(1, '2025-07-17 09:05:00', 'view'),
|
||||
(1, '2025-07-17 09:10:00', 'purchase'),
|
||||
(2, '2025-07-17 10:00:00', 'click'),
|
||||
(2, '2025-07-17 10:05:00', 'click'),
|
||||
(2, '2025-07-17 11:00:00', 'view'),
|
||||
(3, '2025-07-16 12:00:00', 'view'),
|
||||
(3, '2025-07-16 12:30:00', 'click'),
|
||||
(3, '2025-07-17 08:00:00', 'purchase'),
|
||||
(4, '2025-07-17 13:00:00', 'click'),
|
||||
(4, '2025-07-17 14:00:00', 'view'),
|
||||
(4, '2025-07-17 15:00:00', 'view'),
|
||||
(5, '2025-07-16 20:00:00', 'view'),
|
||||
(5, '2025-07-17 01:00:00', 'click'),
|
||||
(5, '2025-07-17 06:00:00', 'purchase'),
|
||||
(6, '2025-07-17 07:30:00', 'view'),
|
||||
(6, '2025-07-17 08:45:00', 'click'),
|
||||
(7, '2025-07-16 23:59:59', 'click'),
|
||||
(7, '2025-07-17 00:01:00', 'purchase'),
|
||||
(7, '2025-07-17 10:10:00', 'view')
|
||||
--1-2) Добавь колонку с кумулятивным количеством событий по user_id (используй count() OVER ...)
|
||||
select
|
||||
user_id,
|
||||
event_time,
|
||||
event_type,
|
||||
count(*) over (partition by user_id order by event_time) as total_click_exact_moment,
|
||||
row_number() over (partition by user_id order by event_time) as row_number
|
||||
from mdrozdov.events
|
||||
--3) да, поддерживается, но не обязательно для коммулятивного суммирования строк
|
||||
--4-6) Сделай WITH recent_events AS (...) — все события за последние 24 часа.
|
||||
with mdrozdov.recent_events AS (
|
||||
select *
|
||||
from mdrozdov.events
|
||||
where event_time >= toDateTime('2025-07-18') - interval 1 day
|
||||
), category_list as (
|
||||
select 'click' as event_type, 'interaction' as category
|
||||
union all
|
||||
select 'view', 'interaction'
|
||||
union all
|
||||
select 'purchase', 'conversion'
|
||||
)
|
||||
select
|
||||
user_id,
|
||||
count(*) as total_events_last_24h,
|
||||
count(distinct cl.category) as uniq_categories
|
||||
from recent_events re
|
||||
left join category_list cl
|
||||
using(event_type)
|
||||
group by user_id
|
||||
order by total_events_last_24h desc, user_id asc
|
||||
limit 5
|
||||
@@ -0,0 +1,65 @@
|
||||
--task 1
|
||||
create table mdrozdov.logs3
|
||||
on CLUSTER clickhouse_cluster (
|
||||
log_id Int,
|
||||
ts DateTime
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/logs3',
|
||||
'{replica}'
|
||||
)
|
||||
order by ts
|
||||
|
||||
alter table mdrozdov.logs3
|
||||
add index idx_ts ts type minmax GRANULARITY 2;
|
||||
|
||||
select *
|
||||
from mdrozdov.logs3
|
||||
where ts between '2024-06-02' and '2024-06-03'
|
||||
|
||||
SELECT
|
||||
query,
|
||||
read_rows,
|
||||
result_rows,
|
||||
read_bytes
|
||||
FROM system.query_log
|
||||
WHERE query LIKE '%logs%'
|
||||
ORDER BY event_time DESC
|
||||
LIMIT 1;
|
||||
|
||||
-- task 2
|
||||
create table mdrozdov.clicks
|
||||
on CLUSTER clickhouse_cluster (
|
||||
id UInt64,
|
||||
country String
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/clicks',
|
||||
'{replica}'
|
||||
)
|
||||
order by id
|
||||
|
||||
alter table mdrozdov.clicks
|
||||
add index idx_country country type set(100) GRANULARITY 2
|
||||
|
||||
Explain
|
||||
select *
|
||||
from mdrozdov.clicks
|
||||
where country = 'RU'
|
||||
|
||||
-- task 3
|
||||
create table mdrozdov.users
|
||||
on CLUSTER clickhouse_cluster (
|
||||
id UInt64,
|
||||
email String
|
||||
)
|
||||
engine = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/mdrozdov/users',
|
||||
'{replica}'
|
||||
)
|
||||
order by id
|
||||
|
||||
ALTER TABLE mdrozdov.users
|
||||
ADD INDEX idx_email email TYPE bloom_filter(0.01) GRANULARITY 2;
|
||||
|
||||
SELECT * FROM mdrozdov.users WHERE email = 'alice@mail.com';
|
||||
@@ -0,0 +1,34 @@
|
||||
-- task 1
|
||||
CREATE TABLE dist_table
|
||||
ENGINE = Distributed(
|
||||
'cluster_name', -- нет доступа к конфигу infra clickhouse
|
||||
'mdrozdov',
|
||||
'logs',
|
||||
rand()
|
||||
);
|
||||
--rand() это параметр по которому мы распределяем данные при insert на разные ноды, в данном
|
||||
--случае нам это нужно для равномерности распределения, потому что таблица не требует особых связей или тп
|
||||
|
||||
-- task 2
|
||||
--Создай VIEW top_active_users, который показывает топ-100 пользователей по числу событий
|
||||
CREATE VIEW top_active_users AS
|
||||
select user_id, count(*) as total_events
|
||||
from events
|
||||
group by user_id
|
||||
order by total_events desc
|
||||
limit 100
|
||||
-- не подходит для дашборда с миллионами строк, потому что вычисляется каждый раз при вызове
|
||||
-- для дашбордов лучше подойдет materialized view, тк хранит данные запроса и обновляется автоматически
|
||||
-- при вставке новых данных
|
||||
|
||||
--task 3
|
||||
CREATE MATERIALIZED VIEW
|
||||
to
|
||||
ENGINE = SummingMergeTree
|
||||
order by date
|
||||
mv_clicks_per_day AS
|
||||
select
|
||||
toDate(event_time) as date,
|
||||
count(*) as total_clicks
|
||||
from raw_events
|
||||
group by date
|
||||
@@ -0,0 +1,94 @@
|
||||
-- Измерение пользователей
|
||||
CREATE TABLE core.dim_user (
|
||||
user_id INT PRIMARY KEY,
|
||||
name TEXT,
|
||||
age INT,
|
||||
email TEXT,
|
||||
registration_date DATE
|
||||
)
|
||||
distributed by (user_id);
|
||||
|
||||
-- Измерение курсов
|
||||
CREATE TABLE core.dim_course (
|
||||
course_id INT PRIMARY KEY,
|
||||
title TEXT,
|
||||
category TEXT
|
||||
)
|
||||
distributed by (course_id);
|
||||
|
||||
-- Измерение уроков
|
||||
CREATE TABLE core.dim_lesson (
|
||||
lesson_id INT PRIMARY KEY,
|
||||
title TEXT,
|
||||
duration_min INT,
|
||||
course_id INT REFERENCES core.dim_course(course_id)
|
||||
)
|
||||
distributed by (lesson_id);
|
||||
|
||||
-- Опционально: измерение времени
|
||||
CREATE TABLE core.dim_date (
|
||||
date DATE PRIMARY KEY,
|
||||
year INT,
|
||||
month INT,
|
||||
day INT,
|
||||
weekday TEXT
|
||||
)
|
||||
distributed replicated;
|
||||
|
||||
-- Таблицы фактов:
|
||||
CREATE TABLE core.fact_lesson_views (
|
||||
user_id INT REFERENCES core.dim_user(user_id),
|
||||
lesson_id INT REFERENCES core.dim_lesson(lesson_id),
|
||||
course_id INT REFERENCES core.dim_course(course_id),
|
||||
viewed_at TIMESTAMP
|
||||
)
|
||||
distributed by (course_id);
|
||||
|
||||
CREATE TABLE core.enrollments (
|
||||
user_id INT REFERENCES core.dim_user(user_id),
|
||||
course_id INT REFERENCES core.dim_course(course_id),
|
||||
enrolled_at TIMESTAMP
|
||||
)
|
||||
distributed by (course_id);
|
||||
|
||||
--ETL переброс данных из raw в core
|
||||
INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
|
||||
SELECT DISTINCT
|
||||
id::INT,
|
||||
name::TEXT,
|
||||
age::INT,
|
||||
email::TEXT,
|
||||
registration_date::DATE
|
||||
FROM raw.users;
|
||||
|
||||
INSERT INTO core.dim_course (course_id, title, category)
|
||||
SELECT DISTINCT
|
||||
id::INT,
|
||||
title::TEXT,
|
||||
category::TEXT
|
||||
FROM raw.courses;
|
||||
|
||||
INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
|
||||
SELECT DISTINCT
|
||||
id::INT,
|
||||
title::TEXT,
|
||||
duration_min::INT,
|
||||
course_id::INT
|
||||
FROM raw.lessons;
|
||||
|
||||
-- Заполнение таблицы фактов
|
||||
insert into core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
|
||||
select
|
||||
lw.user_id,
|
||||
lw.lesson_id,
|
||||
l.course_id,
|
||||
lw.viewed_at
|
||||
from raw.lessons_views lw
|
||||
join raw.lessons l on lw.lesson_id = l.id;
|
||||
|
||||
insert into core.enrollments (user_id, course_id, enrolled_at)
|
||||
select
|
||||
user_id,
|
||||
course_id,
|
||||
enrolled_at
|
||||
from raw.enrollments;
|
||||
@@ -0,0 +1,71 @@
|
||||
--lesson_popularity_summary data mart
|
||||
create materialized view dm.lesson_popularity_summary as
|
||||
select
|
||||
dl.lesson_id,
|
||||
dl.title as lesson_title,
|
||||
dl.course_id,
|
||||
dc.title as course_title,
|
||||
count(flv.lesson_id) as total_views,
|
||||
count(distinct flv.user_id) as unique_users,
|
||||
min(flv.viewed_at) as first_view,
|
||||
max(flv.viewed_at) as last_view
|
||||
from core.dim_lesson dl
|
||||
left join core.dim_course dc
|
||||
on dl.course_id = dc.course_id
|
||||
left join core.fact_lesson_views flv
|
||||
on dl.lesson_id = flv.lesson_id
|
||||
group by dl.lesson_id, dl.title, dl.course_id, dc.title
|
||||
|
||||
-- inactive_users_summary data mart
|
||||
create materialized view dm.inactive_users_summary as
|
||||
select
|
||||
du.user_id,
|
||||
du.name as user_name,
|
||||
du.email,
|
||||
du.age,
|
||||
count(e.course_id) as enrollments_count,
|
||||
du.registration_date
|
||||
from core.dim_user du
|
||||
left join core.enrollments e
|
||||
on du.user_id = e.user_id
|
||||
left join core.fact_lesson_views flv
|
||||
on du.user_id = flv.user_id
|
||||
where flv.user_id is null
|
||||
group by du.user_id, du.name, du.email, du.age, du.registration_date
|
||||
|
||||
-- course_completion_rate data mart
|
||||
create materialized view dm.course_completion_rate as
|
||||
with total_lessons_per_course as(
|
||||
select
|
||||
course_id,
|
||||
count(*) as lessons_in_course
|
||||
from core.dim_lesson
|
||||
group by course_id
|
||||
), lessons_viewed as (
|
||||
select
|
||||
user_id,
|
||||
course_id,
|
||||
count(*) as lessons_viewed
|
||||
from fact_lesson_views flv
|
||||
group by user_id, course_id
|
||||
)
|
||||
select
|
||||
du.user_id as user_id,
|
||||
du.name as user_name,
|
||||
dc.course_id as course_id,
|
||||
dc.title as course_title,
|
||||
tlpc.lessons_in_course,
|
||||
coalesce(lv.lessons_viewed, 0) as lessons_viewed,
|
||||
round(
|
||||
coalesce(
|
||||
coalesce(lv.lessons_viewed, 0)*1.0/nullif(tlpc.lessons_in_course, 0), 0
|
||||
), 2) as completion_rate
|
||||
from core.enrollments e
|
||||
join core.dim_user du
|
||||
on e.user_id = du.user_id
|
||||
join total_lessons_per_course tlpc
|
||||
on tlpc.course_id = e.course_id
|
||||
join core.dim_course dc
|
||||
on e.course_id = dc.course_id
|
||||
left join lessons_viewed lv
|
||||
on du.user_id = lv.user_id and dc.course_id = lv.course_id
|
||||
@@ -0,0 +1,44 @@
|
||||
create schema raw;
|
||||
create schema core;
|
||||
create schema dm;
|
||||
|
||||
create table raw.users(
|
||||
id INT,
|
||||
name varchar(50),
|
||||
age INT,
|
||||
email varchar(50),
|
||||
registration_date date
|
||||
)
|
||||
distributed by (id);
|
||||
|
||||
create table raw.courses (
|
||||
id INT,
|
||||
title text,
|
||||
category varchar(40),
|
||||
created_at date
|
||||
)
|
||||
distributed by (id);
|
||||
|
||||
create table raw.lessons (
|
||||
id INT,
|
||||
course_id INT,
|
||||
title TEXT,
|
||||
duration_min INT
|
||||
)
|
||||
distributed by (course_id);
|
||||
|
||||
create table raw.enrollments (
|
||||
id INT,
|
||||
user_id INT,
|
||||
course_id INT,
|
||||
enrolled_at date
|
||||
)
|
||||
distributed by (user_id);
|
||||
|
||||
create table raw.lessons_views(
|
||||
id int,
|
||||
user_id int,
|
||||
lesson_id int,
|
||||
viewed_at timestamp
|
||||
)
|
||||
distributed by (user_id)
|
||||
Reference in New Issue
Block a user