Merge pull request 'added STEP 1, 3, 4.3' (#32) from OS-233 into main
Reviewed-on: https://gitea.de-infra.servehttp.com/de-infrastructure/DWH/pulls/32 Reviewed-by: aleksey bogatyrev <abogatyrev@example.com>
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
--- создаем необх. схемы для DWH
|
||||
|
||||
CREATE SCHEMA raw;
|
||||
CREATE SCHEMA core;
|
||||
CREATE SCHEMA dm;
|
||||
@@ -0,0 +1,44 @@
|
||||
--создаем таблицы для raw слоя с DISTRIBUTED REPLICATED, т.к. таблицы довольно маленькие
|
||||
|
||||
|
||||
CREATE TABLE raw.users (
|
||||
id BIGINT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
age INT,
|
||||
email VARCHAR(255) UNIQUE,
|
||||
registration_date DATE NOT NULL
|
||||
)
|
||||
DISTRIBUTED REPLICATED;
|
||||
|
||||
CREATE TABLE raw.courses(
|
||||
id BIGINT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
category VARCHAR(255) CHECK (category IN ('data', 'programming', 'business')),
|
||||
created_at DATE NOT NULL
|
||||
)
|
||||
DISTRIBUTED REPLICATED;
|
||||
|
||||
CREATE TABLE raw.lessons(
|
||||
id BIGINT PRIMARY KEY,
|
||||
course_id INT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
duration_min INT NOT NULL
|
||||
)
|
||||
DISTRIBUTED REPLICATED;
|
||||
|
||||
|
||||
CREATE TABLE raw.enrollments(
|
||||
id BIGINT PRIMARY KEY,
|
||||
user_id INT NOT NULL,
|
||||
course_id INT NOT NULL,
|
||||
enrolled_at DATE NOT NULL
|
||||
)
|
||||
DISTRIBUTED REPLICATED;
|
||||
|
||||
CREATE TABLE raw.lesson_views(
|
||||
id BIGINT PRIMARY KEY,
|
||||
user_id INT NOT NULL,
|
||||
lesson_id INT NOT NULL,
|
||||
viewed_at TIMESTAMP NOT NULL
|
||||
)
|
||||
DISTRIBUTED REPLICATED;
|
||||
@@ -0,0 +1,95 @@
|
||||
--- ИЗМЕРЕНИЕ ПОЛЬЗОВАТЕЛЕЙ, РАСПРЕДЕЛЕНИЕ ПО user_id - на проде табл. может быть большой
|
||||
|
||||
CREATE TABLE core.dim_user (
|
||||
user_id INT PRIMARY KEY,
|
||||
name TEXT,
|
||||
age INT,
|
||||
email TEXT,
|
||||
registration_date DATE
|
||||
)
|
||||
DISTRIBUTED BY (user_id);
|
||||
|
||||
INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
|
||||
SELECT DISTINCT
|
||||
id::INT,
|
||||
name::TEXT,
|
||||
age::INT,
|
||||
email::TEXT,
|
||||
registration_date::DATE
|
||||
FROM raw.users;
|
||||
|
||||
|
||||
|
||||
--- ИЗМЕРЕНИЕ КУРСОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник
|
||||
|
||||
CREATE TABLE core.dim_course (
|
||||
course_id INT PRIMARY KEY,
|
||||
title TEXT,
|
||||
category TEXT
|
||||
)
|
||||
DISTRIBUTED REPLICATED;
|
||||
|
||||
INSERT INTO core.dim_course (course_id, title, category)
|
||||
SELECT DISTINCT
|
||||
id::BIGINT,
|
||||
title::TEXT,
|
||||
category::VARCHAR(255)
|
||||
FROM raw.courses;
|
||||
|
||||
|
||||
|
||||
--- ИЗМЕРЕНИЕ УРОКОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник
|
||||
|
||||
CREATE TABLE core.dim_lesson (
|
||||
lesson_id INT PRIMARY KEY,
|
||||
title TEXT,
|
||||
duration_min INT,
|
||||
course_id INT REFERENCES core.dim_course(course_id)
|
||||
)
|
||||
DISTRIBUTED REPLICATED;
|
||||
|
||||
INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
|
||||
SELECT DISTINCT
|
||||
id::BIGINT,
|
||||
title::TEXT,
|
||||
duration_min::INT,
|
||||
course_id::INT
|
||||
FROM raw.lessons;
|
||||
|
||||
|
||||
---- ФАКТ ПРОСМОТРОВ УРОКОВ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN
|
||||
CREATE TABLE core.fact_lesson_views (
|
||||
user_id INT REFERENCES core.dim_user(user_id),
|
||||
lesson_id INT REFERENCES core.dim_lesson(lesson_id),
|
||||
course_id INT REFERENCES core.dim_course(course_id),
|
||||
viewed_at TIMESTAMP
|
||||
)
|
||||
DISTRIBUTED BY (user_id);
|
||||
|
||||
INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
|
||||
SELECT
|
||||
lv.user_id::INT,
|
||||
lv.lesson_id::INT,
|
||||
l.course_id::INT,
|
||||
lv.viewed_at::TIMESTAMP
|
||||
FROM raw.lesson_views lv
|
||||
LEFT JOIN raw.lessons l ON lv.lesson_id = l.id;
|
||||
|
||||
|
||||
|
||||
|
||||
---- ФАКТ ЗАПИСИ НА КУРСЫ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN
|
||||
CREATE TABLE core.enrollments (
|
||||
user_id INT REFERENCES core.dim_user(user_id),
|
||||
course_id INT REFERENCES core.dim_course(course_id),
|
||||
enrolled_at TIMESTAMP
|
||||
)
|
||||
DISTRIBUTED BY (user_id);
|
||||
|
||||
|
||||
INSERT INTO core.enrollments (user_id, course_id, enrolled_at)
|
||||
SELECT
|
||||
user_id::INT,
|
||||
course_id::INT,
|
||||
enrolled_at::DATE
|
||||
FROM raw.enrollments
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
- ВИТРИНА АНАЛИЗА ПОПУЛЯРНОСТИ КАЖДОГО УРОКА
|
||||
- ВСЕ УЧАСТВУЮЩИЕ ИЗМЕРЕНИЯ DISTRIBUTED REPLICATED
|
||||
- ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ DISTRIBUTED BY (lesson_id, course_id)
|
||||
*/
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS dm.lesson_popularity_summary AS
|
||||
-- EXPLAIN ANALYZE
|
||||
DISTRIBUTED BY (lesson_id, course_id)
|
||||
WITH t1 AS (
|
||||
SELECT
|
||||
lesson_id,
|
||||
course_id,
|
||||
COUNT(*) AS total_views,
|
||||
COUNT(DISTINCT user_id) AS unique_users,
|
||||
MIN(viewed_at)AS first_view,
|
||||
MAX(viewed_at)AS last_view
|
||||
FROM core.fact_lesson_views
|
||||
GROUP BY lesson_id, course_id
|
||||
),
|
||||
t2 AS (
|
||||
SELECT
|
||||
l.lesson_id,
|
||||
l.title AS lesson_title,
|
||||
c.course_id,
|
||||
c.title AS course_title
|
||||
FROM core.dim_course c
|
||||
LEFT JOIN core.dim_lesson l USING(course_id)
|
||||
)
|
||||
|
||||
SELECT
|
||||
t1.lesson_id,
|
||||
t2.lesson_title,
|
||||
t1.course_id,
|
||||
t2.course_title,
|
||||
t1.total_views,
|
||||
t1.unique_users,
|
||||
t1.first_view,
|
||||
t1.last_view
|
||||
FROM t1
|
||||
LEFT JOIN t2 USING(lesson_id, course_id);
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
- ВИТРИНА АНАЛИЗА НЕАКТИВНЫХ ПОЛЬЗОВАТЕЛЕЙ
|
||||
- ВСЕ УЧАСТВУЮЩИЕ ТАБЛИЦЫ DISTRIBUTED BY (user_id)
|
||||
- ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ ТАК ЖЕ DISTRIBUTED BY (user_id)
|
||||
*/
|
||||
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS dm.inactive_users_summary AS
|
||||
-- EXPLAIN ANALYZE
|
||||
DISTRIBUTED BY (user_id)
|
||||
WITH t1 AS (
|
||||
SELECT
|
||||
user_id,
|
||||
count(DISTINCT course_id) AS enrollments_count
|
||||
FROM core.enrollments
|
||||
GROUP BY user_id
|
||||
)
|
||||
SELECT
|
||||
u.user_id,
|
||||
u.name AS user_name,
|
||||
u.email,
|
||||
u.age,
|
||||
coalesce(t1.enrollments_count, 0) AS enrollments_count,
|
||||
u.registration_date
|
||||
FROM core.dim_user u
|
||||
LEFT JOIN t1 USING (user_id)
|
||||
WHERE u.user_id NOT EXISTS (SELECT user_id FROM core.fact_lesson_views);
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
- ВИТРИНА ДЛЯ АНАЛИЗА КОНВЕРСИИ ПРОХОЖДЕНИЯ КУРСА ПОЛЬЗОВАТЕЛЯМИ
|
||||
- CORE.DIM_USER И CORE.FACT_LESSON_VIEWS DISTRIBUTED BY (user_id)
|
||||
- ИТОГОВОЕ РАСПРЕДЕЛЕНИЕ ПО (user_id, course_id)
|
||||
*/
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS dm.course_completion_rate AS
|
||||
WITH t1 AS (
|
||||
SELECT
|
||||
e.user_id,
|
||||
du.name AS user_name,
|
||||
e.course_id,
|
||||
dc.title AS course_title,
|
||||
COUNT(DISTINCT dl.lesson_id) AS lessons_in_course
|
||||
FROM core.enrollments e
|
||||
JOIN core.dim_user du USING (user_id)
|
||||
JOIN core.dim_course dc USING (course_id)
|
||||
LEFT JOIN core.dim_lesson dl USING (course_id)
|
||||
GROUP BY user_id, du.name, course_id, dc.title
|
||||
),
|
||||
t2 AS (
|
||||
SELECT
|
||||
user_id,
|
||||
course_id,
|
||||
COUNT(DISTINCT lesson_id) AS lessons_viewed
|
||||
FROM core.fact_lesson_views
|
||||
GROUP BY user_id, course_id
|
||||
)
|
||||
|
||||
SELECT
|
||||
t1.user_id,
|
||||
t1.user_name,
|
||||
t1.course_id,
|
||||
t1.course_title,
|
||||
t1.lessons_in_course,
|
||||
COALESCE(t2.lessons_viewed, 0) AS lessons_viewed,
|
||||
COALESCE(ROUND(lessons_viewed::DECIMAL / NULLIF(lessons_in_course, 0), 2), 0) AS completion_rate
|
||||
FROM t1
|
||||
LEFT JOIN t2 USING (user_id, course_id)
|
||||
DISTRIBUTED BY (user_id, course_id);
|
||||
@@ -0,0 +1,3 @@
|
||||
SELECT * FROM dm.lesson_popularity_summary LIMIT 10;
|
||||
SELECT * FROM dm.inactive_users_summary LIMIT 10;
|
||||
SELECT * FROM dm.course_completion_rate LIMIT 10;
|
||||
Reference in New Issue
Block a user