From b9c57af98083476a7d74267265a2cb4605e4bd38 Mon Sep 17 00:00:00 2001 From: SERGEY_KONEV Date: Tue, 23 Dec 2025 21:03:02 +0300 Subject: [PATCH 1/4] added STEP 1, 3, 4.3 --- skonev/greenplum/STEP1.1_create_schemas.sql | 5 + skonev/greenplum/STEP1.2_create_tables.sql | 44 +++++++++ skonev/greenplum/STEP3_create_core_tables.sql | 95 +++++++++++++++++++ .../STEP4.3_dm.course_completion_rate.sql | 39 ++++++++ 4 files changed, 183 insertions(+) create mode 100644 skonev/greenplum/STEP1.1_create_schemas.sql create mode 100644 skonev/greenplum/STEP1.2_create_tables.sql create mode 100644 skonev/greenplum/STEP3_create_core_tables.sql create mode 100644 skonev/greenplum/STEP4.3_dm.course_completion_rate.sql diff --git a/skonev/greenplum/STEP1.1_create_schemas.sql b/skonev/greenplum/STEP1.1_create_schemas.sql new file mode 100644 index 0000000..fc25ae4 --- /dev/null +++ b/skonev/greenplum/STEP1.1_create_schemas.sql @@ -0,0 +1,5 @@ +--- создаем необх. схемы для DWH + +CREATE SCHEMA raw; +CREATE SCHEMA core; +CREATE SCHEMA dm; \ No newline at end of file diff --git a/skonev/greenplum/STEP1.2_create_tables.sql b/skonev/greenplum/STEP1.2_create_tables.sql new file mode 100644 index 0000000..a494b59 --- /dev/null +++ b/skonev/greenplum/STEP1.2_create_tables.sql @@ -0,0 +1,44 @@ +--создаем таблицы для raw слоя с DISTRIBUTED REPLICATED, т.к. таблицы довольно маленькие + + +CREATE TABLE raw.users ( + id BIGINT PRIMARY KEY, + name VARCHAR(100), + age INT, + email VARCHAR(255) UNIQUE, + registration_date DATE NOT NULL +) +DISTRIBUTED REPLICATED; + +CREATE TABLE raw.courses( + id BIGINT PRIMARY KEY, + title TEXT NOT NULL, + category VARCHAR(255) CHECK (category IN ('data', 'programming', 'business')), + created_at DATE NOT NULL + ) + DISTRIBUTED REPLICATED; + +CREATE TABLE raw.lessons( + id BIGINT PRIMARY KEY, + course_id INT NOT NULL, + title TEXT NOT NULL, + duration_min INT NOT NULL + ) + DISTRIBUTED REPLICATED; + + +CREATE TABLE raw.enrollments( + id BIGINT PRIMARY KEY, + user_id INT NOT NULL, + course_id INT NOT NULL, + enrolled_at DATE NOT NULL + ) + DISTRIBUTED REPLICATED; + +CREATE TABLE raw.lesson_views( + id BIGINT PRIMARY KEY, + user_id INT NOT NULL, + lesson_id INT NOT NULL, + viewed_at TIMESTAMP NOT NULL + ) + DISTRIBUTED REPLICATED; diff --git a/skonev/greenplum/STEP3_create_core_tables.sql b/skonev/greenplum/STEP3_create_core_tables.sql new file mode 100644 index 0000000..5e7272a --- /dev/null +++ b/skonev/greenplum/STEP3_create_core_tables.sql @@ -0,0 +1,95 @@ +--- ИЗМЕРЕНИЕ ПОЛЬЗОВАТЕЛЕЙ, РАСПРЕДЕЛЕНИЕ ПО user_id - на проде табл. может быть большой + +CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE +) +DISTRIBUTED BY (user_id); + +INSERT INTO core.dim_user (user_id, name, age, email, registration_date) +SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE +FROM raw.users; + + + +--- ИЗМЕРЕНИЕ КУРСОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник + +CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT +) +DISTRIBUTED REPLICATED; + +INSERT INTO core.dim_course (course_id, title, category) +SELECT DISTINCT + id::BIGINT, + title::TEXT, + category::VARCHAR(255) +FROM raw.courses; + + + +--- ИЗМЕРЕНИЕ УРОКОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник + +CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) +) +DISTRIBUTED REPLICATED; + +INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) +SELECT DISTINCT + id::BIGINT, + title::TEXT, + duration_min::INT, + course_id::INT +FROM raw.lessons; + + +---- ФАКТ ПРОСМОТРОВ УРОКОВ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN +CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP +) +DISTRIBUTED BY (user_id); + +INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) +SELECT + lv.user_id::INT, + lv.lesson_id::INT, + l.course_id::INT, + lv.viewed_at::TIMESTAMP +FROM raw.lesson_views lv +LEFT JOIN raw.lessons l ON lv.lesson_id = l.id; + + + + +---- ФАКТ ЗАПИСИ НА КУРСЫ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN +CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP +) +DISTRIBUTED BY (user_id); + + +INSERT INTO core.enrollments (user_id, course_id, enrolled_at) +SELECT + user_id::INT, + course_id::INT, + enrolled_at::DATE +FROM raw.enrollments \ No newline at end of file diff --git a/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql new file mode 100644 index 0000000..9446542 --- /dev/null +++ b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql @@ -0,0 +1,39 @@ +/# +- ВИТРИНА ДЛЯ АНАЛИЗА КОНВЕРСИИ ПРОХОЖДЕНИЯ КУРСА ПОЛЬЗОВАТЕЛЯМИ +- ИТОГОВОЕ РАСПРЕДЕЛЕНИЕ ПО (user_id, course_id) +#/ + +CREATE MATERIALIZED VIEW dm.course_completion_rate AS +WITH t1 AS ( +SELECT DISTINCT + e.user_id, + du.name AS user_name, + e.course_id, + dc.title AS course_title, + COUNT(DISTINCT dl.lesson_id) AS lessons_in_course +FROM core.enrollments e +JOIN core.dim_user du USING (user_id) +JOIN core.dim_course dc USING (course_id) +LEFT JOIN core.dim_lesson dl USING (course_id) +GROUP BY user_id, du.name, course_id, dc.title +), +t2 AS ( +SELECT + user_id, + course_id, + COUNT(lesson_id) AS lessons_viewed +FROM core.fact_lesson_views +GROUP BY user_id, course_id +) + +SELECT + t1.user_id, + t1.user_name, + t1.course_id, + t1.course_title, + t1.lessons_in_course, + COALESCE(t2.lessons_viewed, 0) AS lessons_viewed, + COALESCE(ROUND(lessons_viewed::DECIMAL / NULLIF(lessons_in_course, 0), 2), 0) AS completion_rate +FROM t1 +LEFT JOIN t2 USING (user_id, course_id) +DISTRIBUTED BY (user_id, course_id); \ No newline at end of file From 6b5ae71123109715c9b516f17d15c557106db8d6 Mon Sep 17 00:00:00 2001 From: SERGEY_KONEV Date: Wed, 24 Dec 2025 22:52:42 +0300 Subject: [PATCH 2/4] fixed according last comments --- skonev/greenplum/STEP4.3_dm.course_completion_rate.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql index 9446542..75793df 100644 --- a/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql +++ b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql @@ -1,11 +1,11 @@ -/# +/* - ВИТРИНА ДЛЯ АНАЛИЗА КОНВЕРСИИ ПРОХОЖДЕНИЯ КУРСА ПОЛЬЗОВАТЕЛЯМИ - ИТОГОВОЕ РАСПРЕДЕЛЕНИЕ ПО (user_id, course_id) -#/ +*/ CREATE MATERIALIZED VIEW dm.course_completion_rate AS WITH t1 AS ( -SELECT DISTINCT +SELECT e.user_id, du.name AS user_name, e.course_id, @@ -21,7 +21,7 @@ t2 AS ( SELECT user_id, course_id, - COUNT(lesson_id) AS lessons_viewed + COUNT(DISTINCT lesson_id) AS lessons_viewed FROM core.fact_lesson_views GROUP BY user_id, course_id ) From ebc2f07fc8caaba89392b93e016ae9a99268451e Mon Sep 17 00:00:00 2001 From: SERGEY_KONEV Date: Thu, 25 Dec 2025 12:25:42 +0300 Subject: [PATCH 3/4] added STEP4.1, STEP4.2, STEP5 --- .../STEP4.1_dm.lesson_popularity_summary.sql | 42 +++++++++++++++++++ .../STEP4.2_dm.inactive_users_summary.sql | 28 +++++++++++++ .../STEP4.3_dm.course_completion_rate.sql | 3 +- skonev/greenplum/STEP5_checking.sql | 3 ++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql create mode 100644 skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql create mode 100644 skonev/greenplum/STEP5_checking.sql diff --git a/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql b/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql new file mode 100644 index 0000000..ed4479e --- /dev/null +++ b/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql @@ -0,0 +1,42 @@ +/* + - ВИТРИНА АНАЛИЗА ПОПУЛЯРНОСТИ КАЖДОГО УРОКА + - ВСЕ УЧАСТВУЮЩИЕ ИЗМЕРЕНИЯ DISTRIBUTED REPLICATED + - ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ DISTRIBUTED BY (lesson_id, course_id) + */ + +CREATE MATERIALIZED VIEW IF NOT EXISTS dm.lesson_popularity_summary AS +-- EXPLAIN ANALYZE +WITH t1 AS ( +SELECT + lesson_id, + course_id, + COUNT(lesson_id) AS total_views, + COUNT(DISTINCT user_id) AS unique_users, + MIN(viewed_at)AS first_view, + MAX(viewed_at)AS last_view +FROM core.fact_lesson_views +GROUP BY lesson_id, course_id +), +t2 AS ( +SELECT + l.lesson_id, + l.title AS lesson_title, + c.course_id, + c.title AS course_title +FROM core.dim_course c +LEFT JOIN core.dim_lesson l USING(course_id) + +) +SELECT + t1.lesson_id, + t2.lesson_title, + t1.course_id, + t2.course_title, + t1.total_views, + t1.unique_users, + t1.first_view, + t1.last_view +FROM t1 +LEFT JOIN t2 USING(lesson_id) + +DISTRIBUTED BY (lesson_id, course_id); \ No newline at end of file diff --git a/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql b/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql new file mode 100644 index 0000000..6ad6475 --- /dev/null +++ b/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql @@ -0,0 +1,28 @@ +/* + - ВИТРИНА АНАЛИЗА НЕАКТИВНЫХ ПОЛЬЗОВАТЕЛЕЙ + - ВСЕ УЧАСТВУЮЩИЕ ТАБЛИЦЫ DISTRIBUTED BY (user_id) + - ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ ТАК ЖЕ DISTRIBUTED BY (user_id) + */ + + +CREATE MATERIALIZED VIEW IF NOT EXISTS dm.inactive_users_summary AS +-- EXPLAIN ANALYZE +WITH t1 AS ( +SELECT + user_id, + count(DISTINCT course_id) AS enrollments_count +FROM core.enrollments +GROUP BY user_id +) +SELECT + u.user_id, + u.name AS user_name, + u.email, + u.age, + coalesce(t1.enrollments_count, 0), + u.registration_date +FROM core.dim_user u +LEFT JOIN t1 USING (user_id) +WHERE u.user_id NOT IN (SELECT user_id FROM core.fact_lesson_views) + +DISTRIBUTED BY (user_id); \ No newline at end of file diff --git a/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql index 75793df..0d2c4de 100644 --- a/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql +++ b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql @@ -1,9 +1,10 @@ /* - ВИТРИНА ДЛЯ АНАЛИЗА КОНВЕРСИИ ПРОХОЖДЕНИЯ КУРСА ПОЛЬЗОВАТЕЛЯМИ +- CORE.DIM_USER И CORE.FACT_LESSON_VIEWS DISTRIBUTED BY (user_id) - ИТОГОВОЕ РАСПРЕДЕЛЕНИЕ ПО (user_id, course_id) */ -CREATE MATERIALIZED VIEW dm.course_completion_rate AS +CREATE MATERIALIZED VIEW IF NOT EXISTS dm.course_completion_rate AS WITH t1 AS ( SELECT e.user_id, diff --git a/skonev/greenplum/STEP5_checking.sql b/skonev/greenplum/STEP5_checking.sql new file mode 100644 index 0000000..106fa4a --- /dev/null +++ b/skonev/greenplum/STEP5_checking.sql @@ -0,0 +1,3 @@ +SELECT * FROM dm.lesson_popularity_summary LIMIT 10; +SELECT * FROM dm.inactive_users_summary LIMIT 10; +SELECT * FROM dm.course_completion_rate LIMIT 10; \ No newline at end of file From 207c435477e0dd25c7a4ca9ed181abeff2a4a0dd Mon Sep 17 00:00:00 2001 From: SERGEY_KONEV Date: Sat, 27 Dec 2025 17:41:27 +0300 Subject: [PATCH 4/4] fixed reltive last comments --- skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql | 8 ++++---- skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql b/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql index ed4479e..9684265 100644 --- a/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql +++ b/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql @@ -6,11 +6,12 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS dm.lesson_popularity_summary AS -- EXPLAIN ANALYZE +DISTRIBUTED BY (lesson_id, course_id) WITH t1 AS ( SELECT lesson_id, course_id, - COUNT(lesson_id) AS total_views, + COUNT(*) AS total_views, COUNT(DISTINCT user_id) AS unique_users, MIN(viewed_at)AS first_view, MAX(viewed_at)AS last_view @@ -25,8 +26,8 @@ SELECT c.title AS course_title FROM core.dim_course c LEFT JOIN core.dim_lesson l USING(course_id) - ) + SELECT t1.lesson_id, t2.lesson_title, @@ -37,6 +38,5 @@ SELECT t1.first_view, t1.last_view FROM t1 -LEFT JOIN t2 USING(lesson_id) +LEFT JOIN t2 USING(lesson_id, course_id); -DISTRIBUTED BY (lesson_id, course_id); \ No newline at end of file diff --git a/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql b/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql index 6ad6475..7b3180f 100644 --- a/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql +++ b/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql @@ -7,6 +7,7 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS dm.inactive_users_summary AS -- EXPLAIN ANALYZE +DISTRIBUTED BY (user_id) WITH t1 AS ( SELECT user_id, @@ -19,10 +20,9 @@ SELECT u.name AS user_name, u.email, u.age, - coalesce(t1.enrollments_count, 0), + coalesce(t1.enrollments_count, 0) AS enrollments_count, u.registration_date FROM core.dim_user u LEFT JOIN t1 USING (user_id) -WHERE u.user_id NOT IN (SELECT user_id FROM core.fact_lesson_views) +WHERE u.user_id NOT EXISTS (SELECT user_id FROM core.fact_lesson_views); -DISTRIBUTED BY (user_id); \ No newline at end of file