Добавил 2 пет проекта

2025-12-05 22:45:12 +04:00
parent 4fb7b2e797
commit 04fe13e9d9
21 changed files with 698 additions and 0 deletions
@@ -0,0 +1,64 @@
+-- 0. Создаём БД
+CREATE DATABASE dwh_db
+    WITH OWNER = tester
+         TEMPLATE = template0
+         ENCODING = 'UTF8';
+
+-- 1. Создём схемы
+CREATE SCHEMA raw;      -- сырые данные
+CREATE SCHEMA core;     -- или core. Тут хранилище: факты и измерения
+CREATE SCHEMA dm;       -- BI-витрины
+
+
+-- 2. Создаём raw-таблицы:
+
+CREATE TABLE raw.users (
+	id serial4 NOT NULL,
+	"name" varchar(100) NOT NULL,
+	age int4 NOT NULL,
+	email varchar(100) NOT NULL,
+	registration_date date NOT NULL,
+	CONSTRAINT users_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.courses (
+	id serial4 NOT NULL,
+	title varchar(100) NOT NULL,
+	category varchar(100) NOT NULL,
+	created_at date NOT NULL,
+	CONSTRAINT courses_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.lessons (
+	id serial4 NOT NULL,
+	course_id int4 NOT NULL,
+	title varchar(100) NOT NULL,
+	duration_min int4 NOT NULL,
+	CONSTRAINT lessons_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+ALTER TABLE raw.lessons ADD CONSTRAINT lessons_course_id_fkey FOREIGN KEY (course_id) REFERENCES raw.courses(id);
+
+CREATE TABLE raw.enrollments (
+	id serial4 NOT NULL,
+	user_id int4 NOT NULL,
+	course_id int4 NOT NULL,
+	enrolled_at date NOT NULL,
+	CONSTRAINT enrollments_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+ALTER TABLE raw.enrollments ADD CONSTRAINT enrollments_course_id_fkey FOREIGN KEY (course_id) REFERENCES raw.courses(id);
+ALTER TABLE raw.enrollments ADD CONSTRAINT enrollments_user_id_fkey FOREIGN KEY (user_id) REFERENCES raw.users(id);
+
+CREATE TABLE raw.lesson_views (
+	id serial4 NOT NULL,
+	user_id int4 NOT NULL,
+	lesson_id int4 NOT NULL,
+	viewed_at timestamp NOT NULL,
+	CONSTRAINT lesson_views_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+ALTER TABLE raw.lesson_views ADD CONSTRAINT lesson_views_lesson_id_fkey FOREIGN KEY (lesson_id) REFERENCES raw.lessons(id);
+ALTER TABLE raw.lesson_views ADD CONSTRAINT lesson_views_user_id_fkey FOREIGN KEY (user_id) REFERENCES raw.users(id);
@@ -0,0 +1,48 @@
+/* Копирую в файлы в контейнер:
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/users_202512050730.csv gpdb1:/tmp/users_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/courses_202512050730.csv gpdb1:/tmp/courses_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/enrollments_202512050730.csv gpdb1:/tmp/enrollments_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/lesson_views_202512050730.csv gpdb1:/tmp/lesson_views_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/lessons_202512050730.csv gpdb1:/tmp/lessons_202512050730.csv
+*/
+
+
+COPY raw.users
+FROM '/tmp/users_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.courses
+FROM '/tmp/courses_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.lessons
+FROM '/tmp/lessons_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.enrollments
+FROM '/tmp/enrollments_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.lesson_views
+FROM '/tmp/lesson_views_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
@@ -0,0 +1,51 @@
+-- Измерение пользователей
+CREATE TABLE core.dim_user (
+    user_id INT PRIMARY KEY,
+    name TEXT,
+    age INT,
+    email TEXT,
+    registration_date DATE
+)
+DISTRIBUTED BY (user_id);
+
+-- Измерение курсов
+CREATE TABLE core.dim_course (
+    course_id INT PRIMARY KEY,
+    title TEXT,
+    category TEXT
+)
+DISTRIBUTED BY (course_id);
+
+-- Измерение уроков
+CREATE TABLE core.dim_lesson (
+    lesson_id INT PRIMARY KEY,
+    title TEXT,
+    duration_min INT,
+    course_id INT REFERENCES core.dim_course(course_id)
+)
+DISTRIBUTED BY (lesson_id);
+
+-- Опционально: измерение времени
+CREATE TABLE core.dim_date (
+    date DATE PRIMARY KEY,
+    year INT,
+    month INT,
+    day INT,
+    weekday TEXT
+)
+DISTRIBUTED BY (date);
+
+CREATE TABLE core.fact_lesson_views (
+    user_id INT REFERENCES core.dim_user(user_id),
+    lesson_id INT REFERENCES core.dim_lesson(lesson_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    viewed_at TIMESTAMP
+)
+DISTRIBUTED BY (user_id);
+
+CREATE TABLE core.enrollments (
+    user_id INT REFERENCES core.dim_user(user_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    enrolled_at TIMESTAMP
+)
+DISTRIBUTED BY (user_id);
@@ -0,0 +1,49 @@
+BEGIN;
+-- UPDATE с проверкой изменений
+update
+	core.dim_user d
+set
+	name = s.name,
+	age = s.age,
+	email = s.email,
+	registration_date = s.registration_date
+from
+	(
+	select
+		distinct
+        id::INT,
+		name::TEXT,
+		age::INT,
+		email::TEXT,
+		registration_date::DATE
+	from
+		raw.users
+) s
+where
+	d.user_id = s.id
+	and (
+       d.name is distinct from s.name
+	or d.age is distinct from s.age
+	or d.email is distinct from s.email
+	or d.registration_date is distinct from s.registration_date
+  );
+-- INSERT новых строк
+INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
+SELECT
+    s.id,
+    s.name,
+    s.age,
+    s.email,
+    s.registration_date
+FROM (
+    SELECT DISTINCT
+        id::INT,
+        name::TEXT,
+        age::INT,
+        email::TEXT,
+        registration_date::DATE
+    FROM raw.users
+) s
+LEFT JOIN core.dim_user d on d.user_id = s.id
+WHERE d.user_id IS NULL;
+COMMIT;
@@ -0,0 +1,39 @@
+BEGIN;
+-- UPDATE с проверкой изменений
+update
+	core.dim_course dc
+set
+	title = s.title,
+	category = s.category
+from
+	(
+	select
+		distinct
+        id::INT,
+		title::TEXT,
+		category::TEXT
+	from
+		raw.courses
+) s
+where
+	dc.course_id = s.id
+	and (
+       dc.title is distinct from s.title
+	or dc.category is distinct from s.category
+  );
+-- INSERT новых строк
+INSERT INTO core.dim_course (course_id, title, category)
+SELECT
+    s.id,
+    s.title,
+    s.category
+FROM (
+    SELECT DISTINCT
+        id::INT,
+		title::TEXT,
+		category::TEXT
+    FROM raw.courses
+) s
+LEFT JOIN core.dim_course cd on cd.course_id = s.id
+WHERE cd.course_id IS NULL;
+COMMIT;
@@ -0,0 +1,44 @@
+BEGIN;
+-- UPDATE с проверкой изменений
+update
+	core.dim_lesson dl
+set
+	title = l.title,
+	duration_min = l.duration_min,
+	course_id = l.course_id
+from
+	(
+	select distinct
+        id::INT,
+        course_id::INT,
+		title::TEXT,
+		duration_min::INT
+	from
+		raw.lessons
+) l
+where
+	dl.lesson_id = l.id
+	and (
+       dl.title is distinct from l.title
+	or dl.duration_min is distinct from l.duration_min
+	or dl.course_id is distinct from l.course_id
+  );
+-- INSERT новых строк
+INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
+SELECT
+    l.id,
+    l.title,
+    l.duration_min,
+    l.course_id
+FROM (
+	select distinct
+        id::INT,
+        course_id::INT,
+		title::TEXT,
+		duration_min::INT
+	from
+		raw.lessons
+) l
+LEFT JOIN core.dim_lesson dl on dl.lesson_id = l.id
+WHERE dl.lesson_id IS NULL;
+COMMIT;
@@ -0,0 +1,17 @@
+insert into core.enrollments (user_id, course_id, enrolled_at)
+select
+	re.user_id,
+	re.course_id,
+	re.enrolled_at
+from
+	(select DISTINCT
+		user_id::INT,
+		course_id::INT,
+		enrolled_at
+	from
+		raw.enrollments) re
+LEFT join core.enrollments ce on 
+	ce.user_id = re.user_id
+	and ce.course_id = re.course_id
+	and ce.enrolled_at = re.enrolled_at
+WHERE ce.user_id IS NULL;
@@ -0,0 +1,22 @@
+-- INSERT новых строк
+INSERT INTO  core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
+select 
+	l.user_id,
+	l.lesson_id,
+	l.course_id,
+	l.viewed_at 
+from (select
+	lv.user_id,
+	lv.lesson_id,
+	l.course_id,
+	lv.viewed_at
+	from
+		raw.lesson_views lv
+	left join raw.lessons l on lv.lesson_id = l.id
+	) l
+LEFT join core.fact_lesson_views flv on 
+	flv.user_id = l.user_id
+	and flv.lesson_id = l.lesson_id
+	and flv.course_id = l.course_id
+	and flv.viewed_at = l.viewed_at 
+WHERE flv.user_id IS NULL;
@@ -0,0 +1,31 @@
+DROP MATERIALIZED VIEW IF EXISTS dm.lesson_popularity_summary;
+CREATE MATERIALIZED VIEW dm.lesson_popularity_summary
+DISTRIBUTED BY (lesson_id)
+AS
+select
+	t1.lesson_id as lesson_id,
+	dl.title as lesson_title,
+	t1.course_id as course_id,
+	dc.title as course_title,
+	t1.total_views as total_views,
+	t1.unique_users as unique_users,
+	t1.first_view as first_view,
+	t1.last_view as last_view
+from
+	(select 
+		flv.lesson_id,
+		flv.course_id,
+		COUNT(*) as total_views,
+		COUNT(distinct flv.user_id) as unique_users,
+		MIN(flv.viewed_at) as first_view,
+		MAX(flv.viewed_at) as last_view
+	from 
+		core.fact_lesson_views flv
+	group by
+		flv.lesson_id,
+		flv.course_id
+		) as t1
+right join core.dim_lesson dl on dl.lesson_id = t1.lesson_id
+left join core.dim_course dc on t1.course_id = dc.course_id
+ORDER by
+	t1.lesson_id;
@@ -0,0 +1,25 @@
+DROP MATERIALIZED VIEW IF EXISTS dm.inactive_users_summary;
+CREATE MATERIALIZED VIEW dm.inactive_users_summary
+DISTRIBUTED BY (user_id)
+AS
+select
+	du.user_id as user_id,
+	du."name" as user_name,
+	du.email as email,
+	du.age as age,
+	COALESCE(e.enrollments_count, 0) as enrollments_count,
+	du.registration_date as registration_date
+from
+	core.dim_user du
+left join 
+	(select
+		user_id,
+		COUNT(*) as enrollments_count
+	from 
+		core.enrollments
+	group by
+		user_id
+		) e on du.user_id = e.user_id 
+left join core.fact_lesson_views flv on du.user_id = flv.user_id
+where
+	flv.user_id is null;
@@ -0,0 +1,39 @@
+DROP MATERIALIZED VIEW IF EXISTS dm.course_completion_rate;
+CREATE MATERIALIZED VIEW dm.course_completion_rate
+DISTRIBUTED BY (user_id)
+AS
+with activity_users as (
+select
+	e.user_id as user_id,
+	e.course_id as course_id,
+	COUNT(flv.viewed_at) as lessons_viewed
+from
+	core.enrollments e
+left join core.dim_lesson dl on e.course_id = dl.course_id
+left join core.fact_lesson_views flv on e.user_id = flv.user_id and e.course_id = flv.course_id and dl.lesson_id = flv.lesson_id
+group by
+	e.user_id,
+	e.course_id),
+course2 as (	
+	select
+		course_id,
+		COUNT(*) as lessons_in_course
+	from core.dim_lesson
+	group by 
+		course_id)
+select
+	au.user_id as user_id,
+	du."name" as user_name,
+	au.course_id as course_id,
+	dc.title as course_title,
+	c2.lessons_in_course as lessons_in_course,
+	au.lessons_viewed as lessons_viewed,
+	ROUND(au.lessons_viewed * 1.0 / c2.lessons_in_course , 2) as completion_rate
+from
+	activity_users au
+left join core.dim_user du on au.user_id = du.user_id
+left join core.dim_course dc on au.course_id = dc.course_id
+left join course2 c2 on au.course_id = c2.course_id
+order by
+	au.user_id,
+	au.course_id;
@@ -0,0 +1,4 @@
+"id","title","category","created_at"
+1,SQL для начинающих,data,2023-01-01
+2,Python для анализа данных,programming,2023-01-15
+3,BI с нуля,business,2023-03-01
@@ -0,0 +1,6 @@
+"id","user_id","course_id","enrolled_at"
+1,1,1,2023-01-15
+2,1,2,2023-02-01
+3,2,1,2023-01-20
+4,3,2,2023-03-05
+5,4,3,2023-04-01
@@ -0,0 +1,7 @@
+"id","user_id","lesson_id","viewed_at"
+1,1,1,2023-01-16 10:00:00.000
+2,1,2,2023-01-16 10:15:00.000
+3,2,1,2023-01-21 09:00:00.000
+4,3,3,2023-03-06 12:00:00.000
+5,4,4,2023-04-02 10:00:00.000
+6,4,5,2023-04-03 10:00:00.000
@@ -0,0 +1,6 @@
+"id","course_id","title","duration_min"
+1,1,SELECT и FROM,10
+2,1,JOIN,15
+3,2,Pandas,20
+4,3,Основы BI,12
+5,3,Метрики и дашборды,18
@@ -0,0 +1,6 @@
+"id","name","age","email","registration_date"
+1,Alice,25,alice@mail.com,2023-01-10
+2,Bob,30,bob@gmail.com,2023-02-05
+3,Charlie,22,charlie@mail.com,2023-02-20
+4,Diana,28,diana@mail.com,2023-03-01
+5,Ethan,35,ethan@gmail.com,2023-03-10