diff --git a/skonev/greenplum/STEP1.1_create_schemas.sql b/skonev/greenplum/STEP1.1_create_schemas.sql new file mode 100644 index 0000000..fc25ae4 --- /dev/null +++ b/skonev/greenplum/STEP1.1_create_schemas.sql @@ -0,0 +1,5 @@ +--- создаем необх. схемы для DWH + +CREATE SCHEMA raw; +CREATE SCHEMA core; +CREATE SCHEMA dm; \ No newline at end of file diff --git a/skonev/greenplum/STEP1.2_create_tables.sql b/skonev/greenplum/STEP1.2_create_tables.sql new file mode 100644 index 0000000..a494b59 --- /dev/null +++ b/skonev/greenplum/STEP1.2_create_tables.sql @@ -0,0 +1,44 @@ +--создаем таблицы для raw слоя с DISTRIBUTED REPLICATED, т.к. таблицы довольно маленькие + + +CREATE TABLE raw.users ( + id BIGINT PRIMARY KEY, + name VARCHAR(100), + age INT, + email VARCHAR(255) UNIQUE, + registration_date DATE NOT NULL +) +DISTRIBUTED REPLICATED; + +CREATE TABLE raw.courses( + id BIGINT PRIMARY KEY, + title TEXT NOT NULL, + category VARCHAR(255) CHECK (category IN ('data', 'programming', 'business')), + created_at DATE NOT NULL + ) + DISTRIBUTED REPLICATED; + +CREATE TABLE raw.lessons( + id BIGINT PRIMARY KEY, + course_id INT NOT NULL, + title TEXT NOT NULL, + duration_min INT NOT NULL + ) + DISTRIBUTED REPLICATED; + + +CREATE TABLE raw.enrollments( + id BIGINT PRIMARY KEY, + user_id INT NOT NULL, + course_id INT NOT NULL, + enrolled_at DATE NOT NULL + ) + DISTRIBUTED REPLICATED; + +CREATE TABLE raw.lesson_views( + id BIGINT PRIMARY KEY, + user_id INT NOT NULL, + lesson_id INT NOT NULL, + viewed_at TIMESTAMP NOT NULL + ) + DISTRIBUTED REPLICATED; diff --git a/skonev/greenplum/STEP3_create_core_tables.sql b/skonev/greenplum/STEP3_create_core_tables.sql new file mode 100644 index 0000000..5e7272a --- /dev/null +++ b/skonev/greenplum/STEP3_create_core_tables.sql @@ -0,0 +1,95 @@ +--- ИЗМЕРЕНИЕ ПОЛЬЗОВАТЕЛЕЙ, РАСПРЕДЕЛЕНИЕ ПО user_id - на проде табл. может быть большой + +CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE +) +DISTRIBUTED BY (user_id); + +INSERT INTO core.dim_user (user_id, name, age, email, registration_date) +SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE +FROM raw.users; + + + +--- ИЗМЕРЕНИЕ КУРСОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник + +CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT +) +DISTRIBUTED REPLICATED; + +INSERT INTO core.dim_course (course_id, title, category) +SELECT DISTINCT + id::BIGINT, + title::TEXT, + category::VARCHAR(255) +FROM raw.courses; + + + +--- ИЗМЕРЕНИЕ УРОКОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник + +CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) +) +DISTRIBUTED REPLICATED; + +INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) +SELECT DISTINCT + id::BIGINT, + title::TEXT, + duration_min::INT, + course_id::INT +FROM raw.lessons; + + +---- ФАКТ ПРОСМОТРОВ УРОКОВ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN +CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP +) +DISTRIBUTED BY (user_id); + +INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) +SELECT + lv.user_id::INT, + lv.lesson_id::INT, + l.course_id::INT, + lv.viewed_at::TIMESTAMP +FROM raw.lesson_views lv +LEFT JOIN raw.lessons l ON lv.lesson_id = l.id; + + + + +---- ФАКТ ЗАПИСИ НА КУРСЫ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN +CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP +) +DISTRIBUTED BY (user_id); + + +INSERT INTO core.enrollments (user_id, course_id, enrolled_at) +SELECT + user_id::INT, + course_id::INT, + enrolled_at::DATE +FROM raw.enrollments \ No newline at end of file diff --git a/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql b/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql new file mode 100644 index 0000000..9684265 --- /dev/null +++ b/skonev/greenplum/STEP4.1_dm.lesson_popularity_summary.sql @@ -0,0 +1,42 @@ +/* + - ВИТРИНА АНАЛИЗА ПОПУЛЯРНОСТИ КАЖДОГО УРОКА + - ВСЕ УЧАСТВУЮЩИЕ ИЗМЕРЕНИЯ DISTRIBUTED REPLICATED + - ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ DISTRIBUTED BY (lesson_id, course_id) + */ + +CREATE MATERIALIZED VIEW IF NOT EXISTS dm.lesson_popularity_summary AS +-- EXPLAIN ANALYZE +DISTRIBUTED BY (lesson_id, course_id) +WITH t1 AS ( +SELECT + lesson_id, + course_id, + COUNT(*) AS total_views, + COUNT(DISTINCT user_id) AS unique_users, + MIN(viewed_at)AS first_view, + MAX(viewed_at)AS last_view +FROM core.fact_lesson_views +GROUP BY lesson_id, course_id +), +t2 AS ( +SELECT + l.lesson_id, + l.title AS lesson_title, + c.course_id, + c.title AS course_title +FROM core.dim_course c +LEFT JOIN core.dim_lesson l USING(course_id) +) + +SELECT + t1.lesson_id, + t2.lesson_title, + t1.course_id, + t2.course_title, + t1.total_views, + t1.unique_users, + t1.first_view, + t1.last_view +FROM t1 +LEFT JOIN t2 USING(lesson_id, course_id); + diff --git a/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql b/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql new file mode 100644 index 0000000..7b3180f --- /dev/null +++ b/skonev/greenplum/STEP4.2_dm.inactive_users_summary.sql @@ -0,0 +1,28 @@ +/* + - ВИТРИНА АНАЛИЗА НЕАКТИВНЫХ ПОЛЬЗОВАТЕЛЕЙ + - ВСЕ УЧАСТВУЮЩИЕ ТАБЛИЦЫ DISTRIBUTED BY (user_id) + - ИТОГОВОЕ ПРЕДСТВАЛЕНИЕ ТАК ЖЕ DISTRIBUTED BY (user_id) + */ + + +CREATE MATERIALIZED VIEW IF NOT EXISTS dm.inactive_users_summary AS +-- EXPLAIN ANALYZE +DISTRIBUTED BY (user_id) +WITH t1 AS ( +SELECT + user_id, + count(DISTINCT course_id) AS enrollments_count +FROM core.enrollments +GROUP BY user_id +) +SELECT + u.user_id, + u.name AS user_name, + u.email, + u.age, + coalesce(t1.enrollments_count, 0) AS enrollments_count, + u.registration_date +FROM core.dim_user u +LEFT JOIN t1 USING (user_id) +WHERE u.user_id NOT EXISTS (SELECT user_id FROM core.fact_lesson_views); + diff --git a/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql new file mode 100644 index 0000000..0d2c4de --- /dev/null +++ b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql @@ -0,0 +1,40 @@ +/* +- ВИТРИНА ДЛЯ АНАЛИЗА КОНВЕРСИИ ПРОХОЖДЕНИЯ КУРСА ПОЛЬЗОВАТЕЛЯМИ +- CORE.DIM_USER И CORE.FACT_LESSON_VIEWS DISTRIBUTED BY (user_id) +- ИТОГОВОЕ РАСПРЕДЕЛЕНИЕ ПО (user_id, course_id) +*/ + +CREATE MATERIALIZED VIEW IF NOT EXISTS dm.course_completion_rate AS +WITH t1 AS ( +SELECT + e.user_id, + du.name AS user_name, + e.course_id, + dc.title AS course_title, + COUNT(DISTINCT dl.lesson_id) AS lessons_in_course +FROM core.enrollments e +JOIN core.dim_user du USING (user_id) +JOIN core.dim_course dc USING (course_id) +LEFT JOIN core.dim_lesson dl USING (course_id) +GROUP BY user_id, du.name, course_id, dc.title +), +t2 AS ( +SELECT + user_id, + course_id, + COUNT(DISTINCT lesson_id) AS lessons_viewed +FROM core.fact_lesson_views +GROUP BY user_id, course_id +) + +SELECT + t1.user_id, + t1.user_name, + t1.course_id, + t1.course_title, + t1.lessons_in_course, + COALESCE(t2.lessons_viewed, 0) AS lessons_viewed, + COALESCE(ROUND(lessons_viewed::DECIMAL / NULLIF(lessons_in_course, 0), 2), 0) AS completion_rate +FROM t1 +LEFT JOIN t2 USING (user_id, course_id) +DISTRIBUTED BY (user_id, course_id); \ No newline at end of file diff --git a/skonev/greenplum/STEP5_checking.sql b/skonev/greenplum/STEP5_checking.sql new file mode 100644 index 0000000..106fa4a --- /dev/null +++ b/skonev/greenplum/STEP5_checking.sql @@ -0,0 +1,3 @@ +SELECT * FROM dm.lesson_popularity_summary LIMIT 10; +SELECT * FROM dm.inactive_users_summary LIMIT 10; +SELECT * FROM dm.course_completion_rate LIMIT 10; \ No newline at end of file