diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2d99370 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +itikhomirov/.DS_Store +.DS_Store diff --git a/itikhomirov/greenplum/Step 1.1.png b/itikhomirov/greenplum/Step 1.1.png new file mode 100644 index 0000000..ff400a1 Binary files /dev/null and b/itikhomirov/greenplum/Step 1.1.png differ diff --git a/itikhomirov/greenplum/Step 1.2.png b/itikhomirov/greenplum/Step 1.2.png new file mode 100644 index 0000000..2f3417b Binary files /dev/null and b/itikhomirov/greenplum/Step 1.2.png differ diff --git a/itikhomirov/greenplum/Step 2.3.png b/itikhomirov/greenplum/Step 2.3.png new file mode 100644 index 0000000..8183aac Binary files /dev/null and b/itikhomirov/greenplum/Step 2.3.png differ diff --git a/itikhomirov/greenplum/Step 3.4.png b/itikhomirov/greenplum/Step 3.4.png new file mode 100644 index 0000000..d4d84df Binary files /dev/null and b/itikhomirov/greenplum/Step 3.4.png differ diff --git a/itikhomirov/greenplum/Step 3.5.png b/itikhomirov/greenplum/Step 3.5.png new file mode 100644 index 0000000..0865d92 Binary files /dev/null and b/itikhomirov/greenplum/Step 3.5.png differ diff --git a/itikhomirov/greenplum/Step 3.6.png b/itikhomirov/greenplum/Step 3.6.png new file mode 100644 index 0000000..c3ce257 Binary files /dev/null and b/itikhomirov/greenplum/Step 3.6.png differ diff --git a/itikhomirov/greenplum/Step 4.7.png b/itikhomirov/greenplum/Step 4.7.png new file mode 100644 index 0000000..1db2192 Binary files /dev/null and b/itikhomirov/greenplum/Step 4.7.png differ diff --git a/itikhomirov/greenplum/Step 4.8.png b/itikhomirov/greenplum/Step 4.8.png new file mode 100644 index 0000000..67e3fa9 Binary files /dev/null and b/itikhomirov/greenplum/Step 4.8.png differ diff --git a/itikhomirov/greenplum/Step 4.9.png b/itikhomirov/greenplum/Step 4.9.png new file mode 100644 index 0000000..b8d1614 Binary files /dev/null and b/itikhomirov/greenplum/Step 4.9.png differ diff --git a/itikhomirov/greenplum/Step 5.10.png b/itikhomirov/greenplum/Step 5.10.png new file mode 100644 index 0000000..af50c92 Binary files /dev/null and b/itikhomirov/greenplum/Step 5.10.png differ diff --git a/itikhomirov/greenplum/pet_gp_infra.sql b/itikhomirov/greenplum/pet_gp_infra.sql new file mode 100644 index 0000000..199fbe4 --- /dev/null +++ b/itikhomirov/greenplum/pet_gp_infra.sql @@ -0,0 +1,241 @@ +-- Этап 1. Инициализация проекта +-- Задание 1: Создай схемы + +CREATE SCHEMA raw; +CREATE SCHEMA core; +CREATE SCHEMA dm; + +-- Задание 2: Создай raw-таблицы + +CREATE TABLE raw.users ( + id int, + name varchar(15), + age int, + email varchar(30), + registration_date DATE +) +DISTRIBUTED BY (id); + +CREATE TABLE raw.courses ( + id int, + title text, + category varchar(30), + created_at DATE +) +DISTRIBUTED BY (id); + +CREATE TABLE raw.lessons ( + id int, + course_id int, + title text, + duration_min int +) +DISTRIBUTED BY (id); + +CREATE TABLE raw.enrollments ( + id int, + user_id int, + course_id int, + enrolled_at DATE +) +DISTRIBUTED BY (id); + +CREATE TABLE raw.lesson_views ( + id int, + user_id int, + lesson_id int, + viewed_at timestamp +) +DISTRIBUTED BY (id); + +-- Этап 2. Загрузка данных (эмуляция Airflow) +-- Задание 3: Выгрузить данные из PostgreSQL и Загрузить CSV в Greenplum + + +-- Экспортировал данные из PostgreSQL в CSV-файлы и загрузил их в Greenplum + +-- Этап 3. Слой Core — обогащённые данные +-- Задание 4: Измерения (dimension tables) + +-- Измерение пользователей +CREATE TABLE core.dim_user ( + user_id INT PRIMARY KEY, + name TEXT, + age INT, + email TEXT, + registration_date DATE +); + +-- Измерение курсов +CREATE TABLE core.dim_course ( + course_id INT PRIMARY KEY, + title TEXT, + category TEXT +); + +-- Измерение уроков +CREATE TABLE core.dim_lesson ( + lesson_id INT PRIMARY KEY, + title TEXT, + duration_min INT, + course_id INT REFERENCES core.dim_course(course_id) +); + +-- Опционально: измерение времени +CREATE TABLE core.dim_date ( + date DATE PRIMARY KEY, + year INT, + month INT, + day INT, + weekday TEXT +); + +-- Задание 5: Факт (fact table) + +CREATE TABLE core.fact_lesson_views ( + user_id INT REFERENCES core.dim_user(user_id), + lesson_id INT REFERENCES core.dim_lesson(lesson_id), + course_id INT REFERENCES core.dim_course(course_id), + viewed_at TIMESTAMP +); + +CREATE TABLE core.enrollments ( + user_id INT REFERENCES core.dim_user(user_id), + course_id INT REFERENCES core.dim_course(course_id), + enrolled_at TIMESTAMP +); + +-- Задание 6: ETL переброса данных из raw в core + +INSERT INTO core.dim_user (user_id, name, age, email, registration_date) -- ok +SELECT DISTINCT + id::INT, + name::TEXT, + age::INT, + email::TEXT, + registration_date::DATE +FROM raw.users; + +INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) -- ok +SELECT DISTINCT + id::INT, + title::TEXT, + duration_min::INT, + course_id::INT +FROM raw.lessons; + +INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) --заполнить из нескольких таблиц с помощью join +SELECT DISTINCT + lv.user_id::INT, + lv.lesson_id::INT, + le.course_id::INT, + lv.viewed_at::TIMESTAMP +FROM raw.lesson_views AS lv +JOIN raw.lessons AS le ON lv.lesson_id = le.id; + +INSERT INTO core.enrollments (user_id, course_id, enrolled_at) -- ok +SELECT DISTINCT + user_id::INT, + course_id::INT, + enrolled_at::DATE +FROM raw.enrollments; + +INSERT INTO core.dim_course (course_id, title, category) -- ok +SELECT DISTINCT + id::INT, + title::TEXT, + category::VARCHAR +FROM raw.courses; + +INSERT INTO core.dim_date (date, year, month, day, weekday) +SELECT DISTINCT + d::DATE, + EXTRACT(YEAR FROM d)::INT, + EXTRACT(MONTH FROM d)::INT, + EXTRACT(DAY FROM d)::INT, + TO_CHAR(d, 'Day')::TEXT +FROM ( + SELECT registration_date AS d FROM raw.users + UNION + SELECT enrolled_at FROM raw.enrollments + UNION + SELECT viewed_at FROM raw.lesson_views +) all_dates +WHERE d IS NOT NULL; + +-- Надо сделать, чтобы вместо номера месяца выводилось название месяца + +ALTER TABLE core.dim_date +ALTER COLUMN month TYPE text; + +UPDATE core.dim_date +SET month = TO_CHAR(date, 'Month'); + +-- Этап 4: Подготовка витрин в Greenplum +-- Задание 7: Витрина dm.lesson_popularity_summary + +CREATE MATERIALIZED VIEW dm.lesson_popularity_summary AS +SELECT + dl.lesson_id, + dl.title AS lesson_title, + dl.course_id, + dc.title AS course_title, + COUNT(flv.user_id) AS total_views, + COUNT(DISTINCT flv.user_id) AS unique_users, + MIN(flv.viewed_at) AS first_view, + MAX(flv.viewed_at) AS last_view +FROM core.fact_lesson_views AS flv +JOIN core.dim_lesson AS dl ON flv.lesson_id = dl.lesson_id +JOIN core.dim_course AS dc ON dl.course_id = dc.course_id +GROUP BY dl.lesson_id, dl.title, dl.course_id, dc.title +ORDER BY dl.lesson_id ASC; + +-- Задание 8: Витрина dm.inactive_users_summary + +CREATE MATERIALIZED VIEW dm.inactive_users_summary AS +SELECT + du.user_id, + du.name AS user_name, + du.email, + du.age, + COUNT(e.course_id) AS enrollments_count, + du.registration_date +FROM core.dim_user AS du +LEFT JOIN core.enrollments AS e ON du.user_id = e.user_id +LEFT JOIN core.fact_lesson_views AS flv ON du.user_id = flv.user_id +WHERE flv.user_id IS NULL +GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date; + +-- Задание 9: Витрина dm.course_completion_rate + +CREATE MATERIALIZED VIEW dm.course_completion_rate AS +SELECT + du.user_id, + du.name AS user_name, + dc.course_id, + dc.title AS course_title, + COUNT(DISTINCT dl.lesson_id) AS lesson_in_course, + COUNT(DISTINCT flv.lesson_id) AS lessons_viewed, + ROUND( + COALESCE( + NULLIF(COUNT(DISTINCT flv.lesson_id)::NUMERIC,0) / NULLIF(COUNT(DISTINCT dl.lesson_id), 0), + 0 + ) ,2 + ) AS completion_rate +FROM core.dim_user AS du +JOIN core.enrollments AS e ON du.user_id = e.user_id +JOIN core.dim_course AS dc ON e.course_id= dc.course_id +JOIN core.dim_lesson AS dl ON dc.course_id = dl.course_id +LEFT JOIN core.fact_lesson_views AS flv ON du.user_id = flv.user_id AND dc.course_id = flv.course_id +GROUP BY du.user_id, user_name, dc.course_id, course_title +ORDER BY du.user_id ASC; + +-- 🧪 Этап 5. Проверка +-- Задание 10: Проверь витрины + +SELECT * FROM dm.lesson_popularity_summary LIMIT 10; +SELECT * FROM dm.inactive_users_summary LIMIT 10; +SELECT * FROM dm.course_completion_rate ORDER BY user_id ASC LIMIT 10; + + +