убрал : из папок

2026-05-28 18:00:56 +03:00
parent e0b74cdc13
commit f045be8a39
30 changed files with 0 additions and 0 deletions
@@ -0,0 +1,46 @@
+-- 1
+CREATE SCHEMA raw;
+CREATE SCHEMA core;
+CREATE SCHEMA dm;
+
+-- 2
+--DROP TABLE IF EXISTS raw.users;
+CREATE TABLE raw.users (
+	id bigint NOT NULL,
+	name varchar(100),
+	age SMALLINT,
+	email varchar(100),
+	registration_date timestamptz
+);
+
+--DROP TABLE IF EXISTS raw.courses;
+CREATE TABLE raw.courses (
+	id bigint NOT NULL,
+	title varchar(150),
+	category varchar(50),
+	created_at timestamptz
+);
+
+--DROP TABLE IF EXISTS raw.lessons;
+CREATE TABLE raw.lessons (
+	id bigint NOT NULL,
+	course_id bigint,
+	title varchar(150),
+	duration_min SMALLINT
+);
+
+--DROP TABLE IF EXISTS raw.enrollments;
+CREATE TABLE raw.enrollments (
+	id bigint NOT NULL,
+	user_id bigint,
+	course_id bigint,
+	enrolled_at timestamptz
+);
+
+--DROP TABLE IF EXISTS raw.lesson_views;
+CREATE TABLE raw.lesson_views (
+	id bigint NOT NULL,
+	user_id bigint,
+	lesson_id bigint,
+	viewed_at timestamptz
+);
@@ -0,0 +1,6 @@
+"id","title","category","created_at"
+1,SQL для начинающих,data,2023-01-01 00:00:00.000 +0300
+2,Python для анализа данных,programming,2023-01-15 00:00:00.000 +0300
+3,BI с нуля,business,2023-03-01 00:00:00.000 +0300
+4,Excel для аналитиков,business,2023-02-01 00:00:00.000 +0300
+5,Основы машинного обучения,data,2023-03-15 00:00:00.000 +0300
@@ -0,0 +1,46 @@
+"id","user_id","course_id","enrolled_at"
+1,1,1,2023-01-15 00:00:00.000 +0300
+2,1,2,2023-02-01 00:00:00.000 +0300
+3,2,1,2023-01-20 00:00:00.000 +0300
+4,3,2,2023-03-05 00:00:00.000 +0300
+5,4,3,2023-04-01 00:00:00.000 +0300
+6,6,1,2023-01-06 00:00:00.000 +0300
+7,6,2,2023-01-10 00:00:00.000 +0300
+8,6,4,2023-02-05 00:00:00.000 +0300
+9,7,1,2023-01-13 00:00:00.000 +0300
+10,7,3,2023-03-10 00:00:00.000 +0300
+11,8,1,2023-01-22 00:00:00.000 +0300
+12,8,2,2023-01-22 00:00:00.000 +0300
+13,8,3,2023-01-22 00:00:00.000 +0300
+14,9,4,2023-02-12 00:00:00.000 +0300
+15,10,1,2023-04-16 00:00:00.000 +0300
+16,11,2,2023-04-21 00:00:00.000 +0300
+17,12,1,2023-01-16 00:00:00.000 +0300
+18,13,2,2023-01-19 00:00:00.000 +0300
+19,13,4,2023-02-20 00:00:00.000 +0300
+20,14,1,2023-01-09 00:00:00.000 +0300
+21,14,2,2023-01-09 00:00:00.000 +0300
+22,14,3,2023-01-09 00:00:00.000 +0300
+23,14,4,2023-02-01 00:00:00.000 +0300
+24,14,5,2023-03-20 00:00:00.000 +0300
+25,15,1,2023-01-11 00:00:00.000 +0300
+26,15,3,2023-03-05 00:00:00.000 +0300
+27,16,2,2023-02-16 00:00:00.000 +0300
+28,16,4,2023-02-16 00:00:00.000 +0300
+29,17,3,2023-03-22 00:00:00.000 +0300
+30,17,5,2023-03-25 00:00:00.000 +0300
+31,18,1,2023-04-02 00:00:00.000 +0300
+32,19,1,2023-04-11 00:00:00.000 +0300
+33,19,2,2023-04-11 00:00:00.000 +0300
+34,20,2,2023-04-19 00:00:00.000 +0300
+35,5,5,2023-03-20 00:00:00.000 +0300
+36,1,4,2023-02-10 00:00:00.000 +0300
+37,2,3,2023-03-15 00:00:00.000 +0300
+38,3,1,2023-02-25 00:00:00.000 +0300
+39,4,2,2023-03-10 00:00:00.000 +0300
+40,6,3,2023-03-08 00:00:00.000 +0300
+41,7,2,2023-02-01 00:00:00.000 +0300
+42,9,1,2023-02-15 00:00:00.000 +0300
+43,12,3,2023-03-10 00:00:00.000 +0300
+44,15,2,2023-02-01 00:00:00.000 +0300
+45,18,4,2023-04-05 00:00:00.000 +0300
@@ -0,0 +1,59 @@
+"id","user_id","lesson_id","viewed_at"
+1,1,1,2023-01-16 10:00:00.000 +0300
+2,1,2,2023-01-16 10:15:00.000 +0300
+3,1,3,2023-01-17 09:00:00.000 +0300
+4,1,4,2023-01-17 09:30:00.000 +0300
+5,1,4,2023-01-17 11:00:00.000 +0300
+6,1,4,2023-01-18 10:00:00.000 +0300
+7,1,6,2023-02-02 14:00:00.000 +0300
+8,1,12,2023-02-15 10:00:00.000 +0300
+9,1,13,2023-02-15 10:30:00.000 +0300
+10,2,1,2023-01-21 09:00:00.000 +0300
+11,2,9,2023-03-16 10:00:00.000 +0300
+12,2,10,2023-03-16 10:30:00.000 +0300
+13,3,6,2023-03-06 12:00:00.000 +0300
+14,3,1,2023-02-26 11:00:00.000 +0300
+15,3,2,2023-02-26 11:20:00.000 +0300
+16,4,9,2023-04-02 10:00:00.000 +0300
+17,4,10,2023-04-03 10:00:00.000 +0300
+18,4,6,2023-03-12 14:00:00.000 +0300
+19,4,7,2023-03-13 09:00:00.000 +0300
+20,5,15,2023-03-21 10:00:00.000 +0300
+21,5,16,2023-03-22 11:00:00.000 +0300
+22,6,1,2023-01-07 10:00:00.000 +0300
+23,6,2,2023-01-07 10:15:00.000 +0300
+24,6,3,2023-01-08 09:00:00.000 +0300
+25,6,4,2023-01-08 09:30:00.000 +0300
+26,6,5,2023-01-09 10:00:00.000 +0300
+27,6,6,2023-01-11 14:00:00.000 +0300
+28,6,7,2023-01-12 10:00:00.000 +0300
+29,6,8,2023-01-13 11:00:00.000 +0300
+30,6,12,2023-02-06 09:00:00.000 +0300
+31,6,13,2023-02-06 09:30:00.000 +0300
+32,6,14,2023-02-07 10:00:00.000 +0300
+33,6,9,2023-03-09 11:00:00.000 +0300
+34,7,1,2023-01-14 10:00:00.000 +0300
+35,7,2,2023-01-14 10:20:00.000 +0300
+36,7,3,2023-01-15 09:00:00.000 +0300
+37,7,9,2023-03-11 14:00:00.000 +0300
+38,7,10,2023-03-11 14:30:00.000 +0300
+39,9,12,2023-02-13 10:00:00.000 +0300
+40,9,13,2023-02-13 10:30:00.000 +0300
+41,9,1,2023-02-16 11:00:00.000 +0300
+42,12,1,2023-01-17 10:00:00.000 +0300
+43,12,2,2023-01-17 10:20:00.000 +0300
+44,12,1,2023-04-10 11:00:00.000 +0300
+45,12,3,2023-04-10 11:30:00.000 +0300
+46,12,9,2023-03-12 10:00:00.000 +0300
+47,13,6,2023-01-20 14:00:00.000 +0300
+48,13,7,2023-01-21 10:00:00.000 +0300
+49,13,12,2023-02-21 09:00:00.000 +0300
+50,14,1,2023-01-09 08:00:00.000 +0300
+51,14,2,2023-01-09 08:15:00.000 +0300
+52,14,3,2023-01-10 10:00:00.000 +0300
+53,14,4,2023-01-10 10:30:00.000 +0300
+54,14,5,2023-01-11 09:00:00.000 +0300
+55,14,6,2023-01-10 14:00:00.000 +0300
+56,14,7,2023-01-11 10:00:00.000 +0300
+57,14,8,2023-01-12 11:00:00.000 +0300
+58,14,9,2023-01-10 08:00:00.000 +0300
@@ -0,0 +1,19 @@
+"id","course_id","title","duration_min"
+1,1,SELECT и FROM,10
+2,1,JOIN,15
+3,1,WHERE и фильтрация,12
+4,1,Агрегации и GROUP BY,18
+5,1,Подзапросы,25
+6,2,Pandas основы,20
+7,2,NumPy,15
+8,2,Визуализация данных,22
+9,3,Основы BI,12
+10,3,Метрики и дашборды,18
+11,3,Power BI введение,20
+12,4,Формулы в Excel,15
+13,4,Сводные таблицы,18
+14,4,Графики и диаграммы,14
+15,5,Введение в ML,20
+16,5,Линейная регрессия,30
+17,5,Классификация,28
+18,5,Кластеризация,25
@@ -0,0 +1,21 @@
+"id","name","age","email","registration_date"
+1,Alice,25,alice@mail.com,2023-01-10 00:00:00.000 +0300
+2,Bob,30,bob@gmail.com,2023-02-05 00:00:00.000 +0300
+3,Charlie,22,charlie@mail.com,2023-02-20 00:00:00.000 +0300
+4,Diana,28,diana@mail.com,2023-03-01 00:00:00.000 +0300
+5,Ethan,35,ethan@gmail.com,2023-03-10 00:00:00.000 +0300
+6,Frank,29,frank@mail.com,2023-01-05 00:00:00.000 +0300
+7,Grace,26,grace@gmail.com,2023-01-12 00:00:00.000 +0300
+8,Henry,32,henry@mail.com,2023-01-20 00:00:00.000 +0300
+9,Iris,24,iris@gmail.com,2023-02-10 00:00:00.000 +0300
+10,Jack,27,jack@mail.com,2023-04-15 00:00:00.000 +0300
+11,Kelly,23,kelly@gmail.com,2023-04-20 00:00:00.000 +0300
+12,Leo,31,leo@mail.com,2023-01-15 00:00:00.000 +0300
+13,Mia,26,mia@gmail.com,2023-01-18 00:00:00.000 +0300
+14,Nina,28,nina@mail.com,2023-01-08 00:00:00.000 +0300
+15,Oscar,33,oscar@gmail.com,2023-01-10 00:00:00.000 +0300
+16,Paul,29,paul@mail.com,2023-02-15 00:00:00.000 +0300
+17,Quinn,24,quinn@gmail.com,2023-03-20 00:00:00.000 +0300
+18,Rita,27,rita@mail.com,2023-04-01 00:00:00.000 +0300
+19,Sam,30,sam@gmail.com,2023-04-10 00:00:00.000 +0300
+20,Tina,25,tina@mail.com,2023-04-18 00:00:00.000 +0300
@@ -0,0 +1,189 @@
+-- 4
+--DROP TABLE IF EXISTS core.dim_user;
+CREATE TABLE core.dim_user (
+	user_id bigint PRIMARY KEY,
+	name varchar(100),
+	age smallint,
+	email varchar(100) NOT NULL,
+	registration_date timestamptz NOT NULL
+) DISTRIBUTED BY (user_id);
+
+--DROP TABLE IF EXISTS core.dim_course;
+CREATE TABLE core.dim_course (
+	course_id bigint PRIMARY KEY,
+	title varchar(150) NOT NULL,
+	category varchar(50),
+	created_at timestamptz NOT NULL
+) DISTRIBUTED BY (course_id);
+
+--DROP TABLE IF EXISTS core.dim_lesson;
+CREATE TABLE core.dim_lesson (
+	lesson_id bigint PRIMARY KEY,
+	title varchar(150) NOT NULL,
+	duration_min smallint,
+	course_id bigint NOT NULL REFERENCES core.dim_course(course_id)
+) DISTRIBUTED BY (lesson_id);
+CREATE INDEX idx_dim_lesson_course_id ON core.dim_lesson(course_id);
+
+--DROP TABLE IF EXISTS core.dim_date;
+CREATE TABLE core.dim_date (
+	date date PRIMARY KEY,
+	year int,
+	month int,
+	day int,
+	weekday text
+) DISTRIBUTED BY (date);
+
+
+-- 5
+--DROP TABLE IF EXISTS core.fact_lesson_views;
+CREATE TABLE core.fact_lesson_views (
+	user_id bigint NOT NULL REFERENCES core.dim_user(user_id),
+	lesson_id bigint NOT NULL REFERENCES core.dim_lesson(lesson_id),
+	course_id bigint NOT NULL REFERENCES core.dim_course(course_id),
+	viewed_at timestamptz NOT NULL
+) DISTRIBUTED BY (user_id);
+CREATE INDEX idx_fact_lesson_views_viewed_at ON core.fact_lesson_views(viewed_at);
+CREATE INDEX idx_fact_lesson_views_user_id ON core.fact_lesson_views(user_id);
+CREATE INDEX idx_fact_lesson_views_lesson_id ON core.fact_lesson_views(lesson_id);
+CREATE INDEX idx_fact_lesson_views_course_id ON core.fact_lesson_views(course_id);
+
+--DROP TABLE IF EXISTS core.enrollments;
+CREATE TABLE core.enrollments (
+	user_id bigint NOT NULL REFERENCES core.dim_user(user_id),
+	course_id bigint NOT NULL REFERENCES core.dim_course(course_id),
+	enrolled_at timestamptz NOT NULL
+) DISTRIBUTED BY (user_id);
+CREATE INDEX idx_enrollments_user_id ON core.enrollments(user_id);
+CREATE INDEX idx_enrollments_course_id ON core.enrollments(course_id);
+
+
+--6
+-- dim_user
+INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
+SELECT
+	raw_u.id,
+	raw_u.name,
+	raw_u.age,
+	raw_u.email,
+	raw_u.registration_date
+FROM
+	raw.users raw_u
+WHERE
+	NOT EXISTS (SELECT 1 FROM core.dim_user core_u WHERE core_u.user_id = raw_u.id)
+	AND raw_u.email IS NOT NULL AND raw_u.registration_date IS NOT NULL;
+
+UPDATE core.dim_user core_u
+SET
+	name = raw_u.name,
+	age = raw_u.age,
+	email = raw_u.email,
+	registration_date = raw_u.registration_date
+FROM
+	raw.users raw_u
+WHERE
+	core_u.user_id = raw_u.id
+	AND (
+		core_u.name != raw_u.name OR
+		core_u.age != raw_u.age OR
+		core_u.email != raw_u.email OR
+		core_u.registration_date != raw_u.registration_date
+	);
+
+-- dim_course
+INSERT INTO core.dim_course (course_id, title, category, created_at)
+SELECT
+	raw_c.id,
+	raw_c.title,
+	raw_c.category,
+	raw_c.created_at
+FROM
+	raw.courses raw_c
+WHERE
+	NOT EXISTS (SELECT 1 FROM core.dim_course core_c WHERE core_c.course_id = raw_c.id)
+	AND raw_c.title IS NOT NULL AND raw_c.created_at IS NOT NULL;
+
+UPDATE core.dim_course core_c
+SET
+	title = raw_c.title,
+	category = raw_c.category,
+	created_at = raw_c.created_at
+FROM
+	raw.courses raw_c
+WHERE
+	core_c.course_id = raw_c.id
+	AND (
+		core_c.title != raw_c.title OR
+		core_c.category != raw_c.category OR
+		core_c.created_at != raw_c.created_at
+	);
+
+-- dim_lesson
+INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
+SELECT
+	raw_l.id,
+	raw_l.title,
+	raw_l.duration_min,
+	raw_l.course_id
+FROM
+	raw.lessons raw_l
+WHERE
+	NOT EXISTS (SELECT 1 FROM core.dim_lesson core_l WHERE core_l.lesson_id = raw_l.id)
+	AND raw_l.title IS NOT NULL AND raw_l.course_id IS NOT NULL;
+
+UPDATE core.dim_lesson core_l
+SET
+	title = raw_l.title,
+	duration_min = raw_l.duration_min,
+	course_id = raw_l.course_id -- решил оставить, тк чисто теоретические же урок может начать принадлежать другому курсу
+FROM
+	raw.lessons raw_l
+WHERE
+	core_l.lesson_id = raw_l.id
+	AND (
+		core_l.title != raw_l.title OR
+		core_l.duration_min != raw_l.duration_min OR
+		core_l.course_id != raw_l.course_id
+	);
+
+-- загуглил как заполнять dim_date:
+INSERT INTO core.dim_date (date, year, month, day, weekday)
+SELECT
+	d::date,
+	EXTRACT(YEAR FROM d)::int,
+	EXTRACT(MONTH FROM d)::int,
+	EXTRACT(DAY FROM d)::int,
+	to_char(d, 'day')
+FROM
+	generate_series('2023-01-01'::date, '2030-12-31'::date, INTERVAL '1 day') AS d;
+
+-- fact_lesson_views
+INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
+SELECT
+	raw_lv.user_id,
+	raw_lv.lesson_id,
+	raw_l.course_id,
+	raw_lv.viewed_at
+FROM
+	raw.lesson_views raw_lv
+JOIN
+	raw.lessons raw_l ON raw_lv.lesson_id = raw_l.id
+WHERE
+	raw_lv.viewed_at > (SELECT max(viewed_at) FROM core.fact_lesson_views) -- но при первой загрузке это не сработало, тк max(vied_at) = NULL, как это исправить - я не знаю
+	AND raw_lv.user_id IS NOT NULL AND raw_lv.lesson_id IS NOT NULL
+	AND raw_l.course_id IS NOT NULL AND raw_lv.viewed_at IS NOT NULL;
+
+-- enrollments
+INSERT INTO core.enrollments (user_id, course_id, enrolled_at)
+SELECT
+	raw_e.user_id,
+	raw_e.course_id,
+	raw_e.enrolled_at
+FROM
+	raw.enrollments raw_e
+WHERE
+	NOT EXISTS (SELECT 1 FROM core.enrollments core_e WHERE 
+	core_e.user_id = raw_e.user_id
+	AND core_e.course_id = raw_e.course_id
+	AND core_e.enrolled_at = raw_e.enrolled_at)
+	AND raw_e.user_id IS NOT NULL AND raw_e.course_id IS NOT NULL AND raw_e.enrolled_at IS NOT NULL;
@@ -0,0 +1,99 @@
+-- 7
+--DROP MATERIALIZED VIEW dm.lesson_popularity_summary;
+CREATE MATERIALIZED VIEW dm.lesson_popularity_summary AS
+WITH results_fact_lesson_views AS (
+	SELECT
+		flv.lesson_id,
+		count(flv.user_id) AS total_views,
+		count(DISTINCT flv.user_id) AS unique_users,
+		min(flv.viewed_at)::date AS first_view,
+		max(flv.viewed_at)::date AS last_view
+	FROM
+		core.fact_lesson_views flv
+	GROUP BY
+		flv.lesson_id
+)
+SELECT
+	dl.lesson_id,
+	dl.title AS lesson_title,
+	dl.course_id,
+	dc.title AS course_title,
+	rflv.total_views,
+	rflv.unique_users,
+	rflv.first_view,
+	rflv.last_view
+FROM
+	core.dim_lesson dl
+JOIN
+	core.dim_course dc ON dl.course_id = dc.course_id
+LEFT JOIN
+	results_fact_lesson_views rflv ON dl.lesson_id = rflv.lesson_id;
+
+
+-- 8
+--DROP MATERIALIZED VIEW dm.inactive_users_summary;
+CREATE MATERIALIZED VIEW dm.inactive_users_summary AS
+SELECT
+	du.user_id,
+	du.name AS user_name,
+	du.email,
+	du.age,
+	e.enrollments_count,
+	du.registration_date
+FROM
+	core.dim_user du
+LEFT JOIN (
+	SELECT
+		user_id,
+		count(course_id) AS enrollments_count
+	FROM
+		core.enrollments
+	GROUP BY
+		user_id
+) e ON du.user_id = e.user_id
+WHERE
+	NOT EXISTS (SELECT 1 FROM core.fact_lesson_views flv WHERE flv.user_id = du.user_id);
+-- интересно, что во время проверки плана запроса, когда кусок с enrollments_count был в CTE, план запроса выдавал какую-то хуйню, а когда я засунул его в подзапрос, сразу индексы заработали, и строки начали эффективней читаться
+
+
+-- 9
+--DROP MATERIALIZED VIEW dm.course_completion_rate;
+CREATE MATERIALIZED VIEW dm.course_completion_rate AS
+WITH results_dim_course_lesson AS (
+	SELECT
+		dc.course_id,
+		dc.title,
+		count(lesson_id) AS lessons_in_course
+	FROM
+		core.dim_course dc
+	JOIN
+		core.dim_lesson dl ON dc.course_id = dl.course_id
+	GROUP BY
+		dc.course_id, dc.title
+),
+results_fact_lesson_views AS (
+	SELECT
+		flv.user_id,
+		flv.course_id,
+		count(DISTINCT lesson_id) AS lessons_viewed
+	FROM
+		core.fact_lesson_views flv
+	GROUP BY
+		flv.user_id, flv.course_id
+)
+SELECT
+	du.user_id,
+	du.name AS user_name,
+	rdcl.course_id,
+	rdcl.title AS course_title,
+	rdcl.lessons_in_course,
+	COALESCE(rflv.lessons_viewed, 0) AS lessons_viewed,
+	round(COALESCE(rflv.lessons_viewed, 0)::numeric / NULLIF(rdcl.lessons_in_course, 0), 2) AS completion_rate
+FROM
+	core.dim_user du
+JOIN
+	core.enrollments e ON du.user_id = e.user_id
+JOIN
+	results_dim_course_lesson rdcl ON e.course_id = rdcl.course_id
+LEFT JOIN
+	results_fact_lesson_views rflv ON e.user_id = rflv.user_id AND e.course_id = rflv.course_id;
@@ -0,0 +1,4 @@
+SELECT * FROM dm.lesson_popularity_summary ORDER BY lesson_id LIMIT 10;
+SELECT * FROM dm.inactive_users_summary ORDER BY user_id LIMIT 10;
+SELECT * FROM dm.course_completion_rate ORDER BY user_id, course_id;
+--пришлось добавить order_by для красивого вывода, потому что создать витрины отсортированными не получается