diff --git a/skonev/greenplum/STEP1.1_create_schemas.sql b/skonev/greenplum/STEP1.1_create_schemas.sql new file mode 100644 index 0000000..fc25ae4 --- /dev/null +++ b/skonev/greenplum/STEP1.1_create_schemas.sql @@ -0,0 +1,5 @@ +--- создаем необх. схемы для DWH + +CREATE SCHEMA raw; +CREATE SCHEMA core; +CREATE SCHEMA dm; \ No newline at end of file diff --git a/skonev/greenplum/STEP1.2_create_tables.sql b/skonev/greenplum/STEP1.2_create_tables.sql new file mode 100644 index 0000000..a494b59 --- /dev/null +++ b/skonev/greenplum/STEP1.2_create_tables.sql @@ -0,0 +1,44 @@ +--создаем таблицы для raw слоя с DISTRIBUTED REPLICATED, т.к. таблицы довольно маленькие + + +CREATE TABLE raw.users ( + id BIGINT PRIMARY KEY, + name VARCHAR(100), + age INT, + email VARCHAR(255) UNIQUE, + registration_date DATE NOT NULL +) +DISTRIBUTED REPLICATED; + +CREATE TABLE raw.courses( + id BIGINT PRIMARY KEY, + title TEXT NOT NULL, + category VARCHAR(255) CHECK (category IN ('data', 'programming', 'business')), + created_at DATE NOT NULL + ) + DISTRIBUTED REPLICATED; + +CREATE TABLE raw.lessons( + id BIGINT PRIMARY KEY, + course_id INT NOT NULL, + title TEXT NOT NULL, + duration_min INT NOT NULL + ) + DISTRIBUTED REPLICATED; + + +CREATE TABLE raw.enrollments( + id BIGINT PRIMARY KEY, + user_id INT NOT NULL, + course_id INT NOT NULL, + enrolled_at DATE NOT NULL + ) + DISTRIBUTED REPLICATED; + +CREATE TABLE raw.lesson_views( + id BIGINT PRIMARY KEY, + user_id INT NOT NULL, + lesson_id INT NOT NULL, + viewed_at TIMESTAMP NOT NULL + ) + DISTRIBUTED REPLICATED; diff --git a/skonev/greenplum/STEP3_create_core_tables.sql b/skonev/greenplum/STEP3_create_core_tables.sql new file mode 100644 index 0000000..5e7272a --- /dev/null +++ b/skonev/greenplum/STEP3_create_core_tables.sql @@ -0,0 +1,95 @@ +--- ИЗМЕРЕНИЕ ПОЛЬЗОВАТЕЛЕЙ, РАСПРЕДЕЛЕНИЕ ПО user_id - на проде табл. может быть большой + +CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE +) +DISTRIBUTED BY (user_id); + +INSERT INTO core.dim_user (user_id, name, age, email, registration_date) +SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE +FROM raw.users; + + + +--- ИЗМЕРЕНИЕ КУРСОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник + +CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT +) +DISTRIBUTED REPLICATED; + +INSERT INTO core.dim_course (course_id, title, category) +SELECT DISTINCT + id::BIGINT, + title::TEXT, + category::VARCHAR(255) +FROM raw.courses; + + + +--- ИЗМЕРЕНИЕ УРОКОВ, РАСПРЕДЕЛЕНИЕ С КОПИРОВАНИЕМ НА НОДУ, т.к. таблица-справочник + +CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) +) +DISTRIBUTED REPLICATED; + +INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) +SELECT DISTINCT + id::BIGINT, + title::TEXT, + duration_min::INT, + course_id::INT +FROM raw.lessons; + + +---- ФАКТ ПРОСМОТРОВ УРОКОВ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN +CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP +) +DISTRIBUTED BY (user_id); + +INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) +SELECT + lv.user_id::INT, + lv.lesson_id::INT, + l.course_id::INT, + lv.viewed_at::TIMESTAMP +FROM raw.lesson_views lv +LEFT JOIN raw.lessons l ON lv.lesson_id = l.id; + + + + +---- ФАКТ ЗАПИСИ НА КУРСЫ, РАСПРЕДЕЛЕНИЕ ПО USER_ID ДЛЯ ИСКЛЮЧЕНИЯ MOTION ПРИ JOIN +CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP +) +DISTRIBUTED BY (user_id); + + +INSERT INTO core.enrollments (user_id, course_id, enrolled_at) +SELECT + user_id::INT, + course_id::INT, + enrolled_at::DATE +FROM raw.enrollments \ No newline at end of file diff --git a/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql new file mode 100644 index 0000000..9446542 --- /dev/null +++ b/skonev/greenplum/STEP4.3_dm.course_completion_rate.sql @@ -0,0 +1,39 @@ +/# +- ВИТРИНА ДЛЯ АНАЛИЗА КОНВЕРСИИ ПРОХОЖДЕНИЯ КУРСА ПОЛЬЗОВАТЕЛЯМИ +- ИТОГОВОЕ РАСПРЕДЕЛЕНИЕ ПО (user_id, course_id) +#/ + +CREATE MATERIALIZED VIEW dm.course_completion_rate AS +WITH t1 AS ( +SELECT DISTINCT + e.user_id, + du.name AS user_name, + e.course_id, + dc.title AS course_title, + COUNT(DISTINCT dl.lesson_id) AS lessons_in_course +FROM core.enrollments e +JOIN core.dim_user du USING (user_id) +JOIN core.dim_course dc USING (course_id) +LEFT JOIN core.dim_lesson dl USING (course_id) +GROUP BY user_id, du.name, course_id, dc.title +), +t2 AS ( +SELECT + user_id, + course_id, + COUNT(lesson_id) AS lessons_viewed +FROM core.fact_lesson_views +GROUP BY user_id, course_id +) + +SELECT + t1.user_id, + t1.user_name, + t1.course_id, + t1.course_title, + t1.lessons_in_course, + COALESCE(t2.lessons_viewed, 0) AS lessons_viewed, + COALESCE(ROUND(lessons_viewed::DECIMAL / NULLIF(lessons_in_course, 0), 2), 0) AS completion_rate +FROM t1 +LEFT JOIN t2 USING (user_id, course_id) +DISTRIBUTED BY (user_id, course_id); \ No newline at end of file