Merge branch 'main' into OS-233

.gitignore

@@ -0,0 +1,2 @@
+itikhomirov/.DS_Store
+.DS_Store

APolikanov/1 - Пет-проект_Онлайн-курсы/1.1 DDL_CREATE_TABLE.SQL

@@ -0,0 +1,50 @@
+DROP SCHEMA IF EXISTS edu_platform CASCADE;
+CREATE SCHEMA edu_platform;
+
+drop table if exists edu_platform.users;
+CREATE TABLE edu_platform.users (
+    id SERIAL PRIMARY KEY,
+    name VARCHAR(100) NOT NULL,
+    age INT NOT NULL,
+    email VARCHAR(100) NOT NULL, 
+    registration_date DATE NOT NULL
+);
+
+drop table if exists edu_platform.courses;
+CREATE TABLE edu_platform.courses (
+	id SERIAL PRIMARY KEY,
+	title VARCHAR(100) NOT NULL,
+	category VARCHAR(100) NOT NULL,
+    created_at DATE not NULL
+);
+
+drop table if exists edu_platform.lessons;
+create TABLE edu_platform.lessons (
+	id SERIAL PRIMARY KEY,
+	course_id INT NOT NULL,
+	title VARCHAR(100) NOT NULL,
+	duration_min INT NOT null,
+	FOREIGN KEY (course_id) REFERENCES edu_platform.courses(id)
+);
+
+drop table if exists edu_platform.enrollments;
+create TABLE edu_platform.enrollments (
+	id SERIAL PRIMARY KEY,
+	user_id	INT NOT NULL,
+	course_id INT NOT NULL,
+	enrolled_at DATE not null,
+	foreign key (user_id) references edu_platform.users(id),
+	foreign key (course_id) references edu_platform.courses(id)
+);
+
+drop table if exists edu_platform.lesson_views;
+create table edu_platform.lesson_views (
+	id SERIAL PRIMARY KEY,
+	user_id	INT NOT NULL,
+	lesson_id INT NOT NULL,
+	viewed_at TIMESTAMP not null,
+	foreign key (user_id) references edu_platform.users(id),
+	foreign key (lesson_id) references edu_platform.lessons(id)
+);
+
+

APolikanov/1 - Пет-проект_Онлайн-курсы/1.2 DML_INSERT_DATA.SQL

@@ -0,0 +1,38 @@
+INSERT INTO edu_platform.users (id, name, age, email, registration_date) VALUES 
+(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'),
+(2,'Bob', 30, 'bob@gmail.com', '2023-02-05'),
+(3,'Charlie', 22, 'charlie@mail.com', '2023-02-20'),
+(4,'Diana', 28, 'diana@mail.com', '2023-03-01'),
+(5,'Ethan', 35, 'ethan@gmail.com', '2023-03-10');
+    
+    
+
+INSERT INTO edu_platform.courses (id, title, category, created_at) VALUES
+(1, 'SQL для начинающих', 'data', '2023-01-01'),
+(2, 'Python для анализа данных', 'programming', '2023-01-15'),
+(3, 'BI с нуля', 'business', '2023-03-01');
+
+-- Таблица lessons
+INSERT INTO edu_platform.lessons (id, course_id, title, duration_min) VALUES
+(1, 1, 'SELECT и FROM', 10),
+(2, 1, 'JOIN', 15),
+(3, 2, 'Pandas', 20),
+(4, 3, 'Основы BI', 12),
+(5, 3, 'Метрики и дашборды', 18);
+
+-- Таблица enrollments
+INSERT INTO edu_platform.enrollments (id, user_id, course_id, enrolled_at) VALUES
+(1, 1, 1, '2023-01-15'),
+(2, 1, 2, '2023-02-01'),
+(3, 2, 1, '2023-01-20'),
+(4, 3, 2, '2023-03-05'),
+(5, 4, 3, '2023-04-01');
+
+-- Таблица lesson_views
+INSERT INTO edu_platform.lesson_views (id, user_id, lesson_id, viewed_at) VALUES
+(1, 1, 1, '2023-01-16 10:00:00'),
+(2, 1, 2, '2023-01-16 10:15:00'),
+(3, 2, 1, '2023-01-21 09:00:00'),
+(4, 3, 3, '2023-03-06 12:00:00'),
+(5, 4, 4, '2023-04-02 10:00:00'),
+(6, 4, 5, '2023-04-03 10:00:00');

APolikanov/1 - Пет-проект_Онлайн-курсы/2.1 DDL_CREATE_MAT_VIEW_user_activity_summary.sql

@@ -0,0 +1,74 @@
+-- Активность пользователей
+
+-- До анализа EXPLAIN:
+drop  Materialized VIEW if exists edu_platform.user_activity_summary;
+create Materialized VIEW edu_platform.user_activity_summary as 
+select
+	u.id as user_id ,
+	u.name as name,
+	COUNT(distinct e.course_id) as enrollment_count,
+	COUNT(distinct lv.lesson_id) as viewed_lessons_count,
+	MAX(lv.viewed_at) as last_view,
+	MIN(lv.viewed_at) as first_view,
+	ROUND(
+		COUNT(DISTINCT l.id) * 100.0 / NULLIF(COUNT(distinct l2.id), 0), 
+		2) as engagement_rate
+from
+	edu_platform.users u 
+left join edu_platform.enrollments e on u.id = e.user_id
+inner join edu_platform.lesson_views lv on u.id = lv.user_id
+left join edu_platform.lessons l on lv.lesson_id  = l.id
+left join edu_platform.lessons l2 on e.course_id  = l2.course_id
+group by 
+	u.id, u.name;
+
+-- После анализа EXPLAIN
+-- Убрал диблирование строк, возникающее при Join-ах.
+-- Минимизировал DISTINCT.
+
+
+drop  Materialized VIEW if exists edu_platform.user_activity_summary;
+create Materialized VIEW edu_platform.user_activity_summary as 
+WITH 
+course_lessons AS (
+    SELECT course_id, COUNT(*) AS lesson_count
+    FROM edu_platform.lessons
+    GROUP BY course_id
+),
+users_enroll as (
+	select
+		u.id as user_id ,
+		u.name as name,
+		COUNT(DISTINCT e.course_id) as enrollment_count,
+		SUM(cl.lesson_count) as lesson_sum
+	from
+		edu_platform.users u
+	left join edu_platform.enrollments e on u.id = e.user_id
+	left join course_lessons cl on e.course_id = cl.course_id
+	group by u.id, u.name
+),
+users_lesson_views as (
+	select
+		lv.user_id,
+		COUNT(distinct lv.lesson_id) as viewed_lessons_count,
+		MAX(lv.viewed_at) as last_view,
+		MIN(lv.viewed_at) as first_view
+	from
+		edu_platform.lesson_views lv
+	group by lv.user_id
+)
+select 
+	ue.user_id,
+	ue.name,
+	ue.enrollment_count,
+	ulv.viewed_lessons_count,
+	ulv.last_view,
+	ulv.first_view,
+	ROUND( (nullif(ulv.viewed_lessons_count, 0) * 100.0) / ue.lesson_sum,
+	2) as engagement_rate
+from users_enroll ue
+left join users_lesson_views ulv on ue.user_id = ulv.user_id;
+
+
+
+

APolikanov/1 - Пет-проект_Онлайн-курсы/2.2 DDL_CREATE_MAT_VIEW_course_summary.sql

@@ -0,0 +1,51 @@
+-- Эффективность курсов
+
+-- После анализа EXPLAIN
+-- Убрал диблирование строк, возникающее при Join-ах.
+-- Минимизировал DISTINCT.
+drop  Materialized VIEW if exists edu_platform.course_summary;
+create Materialized VIEW edu_platform.course_summary as 
+with
+enroll as (
+	select 
+		course_id, 
+		COUNT(DISTINCT user_id) as unique_users -- Может ли один и тот же юзер записаться на курс больше одного раза? Если да, то нужен DISTINCT
+	from 
+		edu_platform.enrollments 
+	group by 
+		course_id),
+lessons as (
+	select 
+		course_id,
+		COUNT(*) as lessons_count
+	from 
+		edu_platform.lessons
+	group by 
+		course_id),
+views as (
+	select 
+		l.course_id as course_id,
+		count(*) as total_views,
+		MIN(lv.viewed_at) as first_view,
+		MAX(lv.viewed_at) as last_view
+	from 
+		edu_platform.lesson_views lv
+	left join edu_platform.lessons l on lv.lesson_id = l.id
+	group by 
+		l.course_id)
+select
+    c.id AS course_id,
+    c.title AS course_title,
+	e.unique_users,
+	l.lessons_count,
+	v.total_views,
+	ROUND( v.total_views * 1.0 / nullif(e.unique_users, 0)
+		,2
+		) as avg_views_per_user,
+	v.first_view,
+	v.last_view
+from
+	edu_platform.courses c
+LEFT JOIN enroll e ON c.id = e.course_id
+LEFT JOIN lessons l ON c.id = l.course_id
+LEFT JOIN views v ON c.id = v.course_id;

APolikanov/1 - Пет-проект_Онлайн-курсы/2.3 DDL_CREATE_MAT_VIEW_platform_summary.sql

@@ -0,0 +1,21 @@
+-- Общая отчетность по взаимодействию
+
+drop  Materialized VIEW if exists edu_platform.platform_summary;
+create Materialized VIEW edu_platform.platform_summary as 
+select
+	(select COUNT(*) from edu_platform.users) as total_users,
+	(select COUNT(*) from edu_platform.courses) as total_courses,
+	(select COUNT(*) from edu_platform.lessons) as total_lessons,
+	(select COUNT(distinct user_id) from edu_platform.lesson_views) as users_with_views,
+	ROUND(
+		(select AVG(t1.cnt) from 
+			(select course_id, COUNT(id) cnt 
+			from edu_platform.lessons 
+			group by course_id) t1) 
+		,2) as avg_lessons_per_course,
+	ROUND(
+		(select AVG(t2.cnt) from
+			(select lesson_id, COUNT(id) cnt 
+			from edu_platform.lesson_views 
+			group by lesson_id) t2)	
+		,2) as avg_views_per_lesson;

APolikanov/2 - GreenPlum-Pet-project/1 DDL_CREATE_SCHEMA_AND_TABLE.sql

@@ -0,0 +1,64 @@
+-- 0. Создаём БД
+CREATE DATABASE dwh_db
+    WITH OWNER = tester
+         TEMPLATE = template0
+         ENCODING = 'UTF8';
+
+-- 1. Создём схемы
+CREATE SCHEMA raw;      -- сырые данные
+CREATE SCHEMA core;     -- или core. Тут хранилище: факты и измерения
+CREATE SCHEMA dm;       -- BI-витрины
+
+
+-- 2. Создаём raw-таблицы:
+
+CREATE TABLE raw.users (
+	id serial4 NOT NULL,
+	"name" varchar(100) NOT NULL,
+	age int4 NOT NULL,
+	email varchar(100) NOT NULL,
+	registration_date date NOT NULL,
+	CONSTRAINT users_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.courses (
+	id serial4 NOT NULL,
+	title varchar(100) NOT NULL,
+	category varchar(100) NOT NULL,
+	created_at date NOT NULL,
+	CONSTRAINT courses_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.lessons (
+	id serial4 NOT NULL,
+	course_id int4 NOT NULL,
+	title varchar(100) NOT NULL,
+	duration_min int4 NOT NULL,
+	CONSTRAINT lessons_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+ALTER TABLE raw.lessons ADD CONSTRAINT lessons_course_id_fkey FOREIGN KEY (course_id) REFERENCES raw.courses(id);
+
+CREATE TABLE raw.enrollments (
+	id serial4 NOT NULL,
+	user_id int4 NOT NULL,
+	course_id int4 NOT NULL,
+	enrolled_at date NOT NULL,
+	CONSTRAINT enrollments_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+ALTER TABLE raw.enrollments ADD CONSTRAINT enrollments_course_id_fkey FOREIGN KEY (course_id) REFERENCES raw.courses(id);
+ALTER TABLE raw.enrollments ADD CONSTRAINT enrollments_user_id_fkey FOREIGN KEY (user_id) REFERENCES raw.users(id);
+
+CREATE TABLE raw.lesson_views (
+	id serial4 NOT NULL,
+	user_id int4 NOT NULL,
+	lesson_id int4 NOT NULL,
+	viewed_at timestamp NOT NULL,
+	CONSTRAINT lesson_views_pkey PRIMARY KEY (id)
+)
+DISTRIBUTED BY (id);
+ALTER TABLE raw.lesson_views ADD CONSTRAINT lesson_views_lesson_id_fkey FOREIGN KEY (lesson_id) REFERENCES raw.lessons(id);
+ALTER TABLE raw.lesson_views ADD CONSTRAINT lesson_views_user_id_fkey FOREIGN KEY (user_id) REFERENCES raw.users(id);

APolikanov/2 - GreenPlum-Pet-project/2 DDM_IMPORT_DATA_TO_RAW.SQL

@@ -0,0 +1,48 @@
+/* Копирую в файлы в контейнер:
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/users_202512050730.csv gpdb1:/tmp/users_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/courses_202512050730.csv gpdb1:/tmp/courses_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/enrollments_202512050730.csv gpdb1:/tmp/enrollments_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/lesson_views_202512050730.csv gpdb1:/tmp/lesson_views_202512050730.csv
+docker cp D:/work/code/DE/GP/Pet-project/IMPORT_DATA/lessons_202512050730.csv gpdb1:/tmp/lessons_202512050730.csv
+*/
+
+
+COPY raw.users
+FROM '/tmp/users_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.courses
+FROM '/tmp/courses_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.lessons
+FROM '/tmp/lessons_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.enrollments
+FROM '/tmp/enrollments_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);
+
+COPY raw.lesson_views
+FROM '/tmp/lesson_views_202512050730.csv'
+WITH (
+    FORMAT csv,
+    HEADER true,
+    DELIMITER ','
+);

APolikanov/2 - GreenPlum-Pet-project/3 DDL_CREATE_CORE.sql

@@ -0,0 +1,51 @@
+-- Измерение пользователей
+CREATE TABLE core.dim_user (
+    user_id INT PRIMARY KEY,
+    name TEXT,
+    age INT,
+    email TEXT,
+    registration_date DATE
+)
+DISTRIBUTED BY (user_id);
+
+-- Измерение курсов
+CREATE TABLE core.dim_course (
+    course_id INT PRIMARY KEY,
+    title TEXT,
+    category TEXT
+)
+DISTRIBUTED BY (course_id);
+
+-- Измерение уроков
+CREATE TABLE core.dim_lesson (
+    lesson_id INT PRIMARY KEY,
+    title TEXT,
+    duration_min INT,
+    course_id INT REFERENCES core.dim_course(course_id)
+)
+DISTRIBUTED BY (lesson_id);
+
+-- Опционально: измерение времени
+CREATE TABLE core.dim_date (
+    date DATE PRIMARY KEY,
+    year INT,
+    month INT,
+    day INT,
+    weekday TEXT
+)
+DISTRIBUTED BY (date);
+
+CREATE TABLE core.fact_lesson_views (
+    user_id INT REFERENCES core.dim_user(user_id),
+    lesson_id INT REFERENCES core.dim_lesson(lesson_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    viewed_at TIMESTAMP
+)
+DISTRIBUTED BY (user_id);
+
+CREATE TABLE core.enrollments (
+    user_id INT REFERENCES core.dim_user(user_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    enrolled_at TIMESTAMP
+)
+DISTRIBUTED BY (user_id);

APolikanov/2 - GreenPlum-Pet-project/6.1 DDM_INSERT_IN_CORE_dim_user.sql

@@ -0,0 +1,49 @@
+BEGIN;
+-- UPDATE с проверкой изменений
+update
+	core.dim_user d
+set
+	name = s.name,
+	age = s.age,
+	email = s.email,
+	registration_date = s.registration_date
+from
+	(
+	select
+		distinct
+        id::INT,
+		name::TEXT,
+		age::INT,
+		email::TEXT,
+		registration_date::DATE
+	from
+		raw.users
+) s
+where
+	d.user_id = s.id
+	and (
+       d.name is distinct from s.name
+	or d.age is distinct from s.age
+	or d.email is distinct from s.email
+	or d.registration_date is distinct from s.registration_date
+  );
+-- INSERT новых строк
+INSERT INTO core.dim_user (user_id, name, age, email, registration_date)
+SELECT
+    s.id,
+    s.name,
+    s.age,
+    s.email,
+    s.registration_date
+FROM (
+    SELECT DISTINCT
+        id::INT,
+        name::TEXT,
+        age::INT,
+        email::TEXT,
+        registration_date::DATE
+    FROM raw.users
+) s
+LEFT JOIN core.dim_user d on d.user_id = s.id
+WHERE d.user_id IS NULL;
+COMMIT;

APolikanov/2 - GreenPlum-Pet-project/6.2 DDM_INSERT_IN_CORE_dim_course.sql

@@ -0,0 +1,39 @@
+BEGIN;
+-- UPDATE с проверкой изменений
+update
+	core.dim_course dc
+set
+	title = s.title,
+	category = s.category
+from
+	(
+	select
+		distinct
+        id::INT,
+		title::TEXT,
+		category::TEXT
+	from
+		raw.courses
+) s
+where
+	dc.course_id = s.id
+	and (
+       dc.title is distinct from s.title
+	or dc.category is distinct from s.category
+  );
+-- INSERT новых строк
+INSERT INTO core.dim_course (course_id, title, category)
+SELECT
+    s.id,
+    s.title,
+    s.category
+FROM (
+    SELECT DISTINCT
+        id::INT,
+		title::TEXT,
+		category::TEXT
+    FROM raw.courses
+) s
+LEFT JOIN core.dim_course cd on cd.course_id = s.id
+WHERE cd.course_id IS NULL;
+COMMIT;

APolikanov/2 - GreenPlum-Pet-project/6.3 DDM_INSERT_IN_CORE_dim_lesson.sql

@@ -0,0 +1,44 @@
+BEGIN;
+-- UPDATE с проверкой изменений
+update
+	core.dim_lesson dl
+set
+	title = l.title,
+	duration_min = l.duration_min,
+	course_id = l.course_id
+from
+	(
+	select distinct
+        id::INT,
+        course_id::INT,
+		title::TEXT,
+		duration_min::INT
+	from
+		raw.lessons
+) l
+where
+	dl.lesson_id = l.id
+	and (
+       dl.title is distinct from l.title
+	or dl.duration_min is distinct from l.duration_min
+	or dl.course_id is distinct from l.course_id
+  );
+-- INSERT новых строк
+INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id)
+SELECT
+    l.id,
+    l.title,
+    l.duration_min,
+    l.course_id
+FROM (
+	select distinct
+        id::INT,
+        course_id::INT,
+		title::TEXT,
+		duration_min::INT
+	from
+		raw.lessons
+) l
+LEFT JOIN core.dim_lesson dl on dl.lesson_id = l.id
+WHERE dl.lesson_id IS NULL;
+COMMIT;

APolikanov/2 - GreenPlum-Pet-project/6.4 DDM_INSERT_IN_CORE_enrollments.sql

@@ -0,0 +1,17 @@
+insert into core.enrollments (user_id, course_id, enrolled_at)
+select
+	re.user_id,
+	re.course_id,
+	re.enrolled_at
+from
+	(select DISTINCT
+		user_id::INT,
+		course_id::INT,
+		enrolled_at
+	from
+		raw.enrollments) re
+LEFT join core.enrollments ce on 
+	ce.user_id = re.user_id
+	and ce.course_id = re.course_id
+	and ce.enrolled_at = re.enrolled_at
+WHERE ce.user_id IS NULL;

APolikanov/2 - GreenPlum-Pet-project/6.5 DDM_INSERT_IN_CORE_fact_lesson_views.sql

@@ -0,0 +1,22 @@
+-- INSERT новых строк
+INSERT INTO  core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at)
+select 
+	l.user_id,
+	l.lesson_id,
+	l.course_id,
+	l.viewed_at 
+from (select
+	lv.user_id,
+	lv.lesson_id,
+	l.course_id,
+	lv.viewed_at
+	from
+		raw.lesson_views lv
+	left join raw.lessons l on lv.lesson_id = l.id
+	) l
+LEFT join core.fact_lesson_views flv on 
+	flv.user_id = l.user_id
+	and flv.lesson_id = l.lesson_id
+	and flv.course_id = l.course_id
+	and flv.viewed_at = l.viewed_at 
+WHERE flv.user_id IS NULL;

APolikanov/2 - GreenPlum-Pet-project/7.1 DDL_CREATE_dm.lesson_popularity_summary.SQL

@@ -0,0 +1,29 @@
+DROP MATERIALIZED VIEW IF EXISTS dm.lesson_popularity_summary;
+CREATE MATERIALIZED VIEW dm.lesson_popularity_summary
+DISTRIBUTED BY (lesson_id)
+AS
+select
+	t1.lesson_id as lesson_id,
+	dl.title as lesson_title,
+	t1.course_id as course_id,
+	dc.title as course_title,
+	t1.total_views as total_views,
+	t1.unique_users as unique_users,
+	t1.first_view as first_view,
+	t1.last_view as last_view
+from
+	(select 
+		flv.lesson_id,
+		flv.course_id,
+		COUNT(*) as total_views,
+		COUNT(distinct flv.user_id) as unique_users,
+		MIN(flv.viewed_at) as first_view,
+		MAX(flv.viewed_at) as last_view
+	from 
+		core.fact_lesson_views flv
+	group by
+		flv.lesson_id,
+		flv.course_id
+		) as t1
+right join core.dim_lesson dl on dl.lesson_id = t1.lesson_id
+left join core.dim_course dc on t1.course_id = dc.course_id;

APolikanov/2 - GreenPlum-Pet-project/7.2 DDL_CREATE_dm.inactive_users_summary.SQL

@@ -0,0 +1,25 @@
+DROP MATERIALIZED VIEW IF EXISTS dm.inactive_users_summary;
+CREATE MATERIALIZED VIEW dm.inactive_users_summary
+DISTRIBUTED BY (user_id)
+AS
+select
+	du.user_id as user_id,
+	du.name as user_name,
+	du.email as email,
+	du.age as age,
+	COALESCE(e.enrollments_count, 0) as enrollments_count,
+	du.registration_date as registration_date
+from
+	core.dim_user du
+left join 
+	(select
+		user_id,
+		COUNT(*) as enrollments_count
+	from 
+		core.enrollments
+	group by
+		user_id
+		) e on du.user_id = e.user_id 
+left join core.fact_lesson_views flv on du.user_id = flv.user_id
+where
+	flv.user_id is null;

APolikanov/2 - GreenPlum-Pet-project/7.3 DDL_CREATE_dm.course_completion_rate.SQL

@@ -0,0 +1,39 @@
+DROP MATERIALIZED VIEW IF EXISTS dm.course_completion_rate;
+CREATE MATERIALIZED VIEW dm.course_completion_rate
+DISTRIBUTED BY (user_id)
+AS
+with activity_users as (
+select
+	e.user_id as user_id,
+	e.course_id as course_id,
+	COUNT(DISTINCT flv.viewed_at) as lessons_viewed
+from
+	core.enrollments e
+left join core.dim_lesson dl on e.course_id = dl.course_id
+left join core.fact_lesson_views flv on e.user_id = flv.user_id and e.course_id = flv.course_id and dl.lesson_id = flv.lesson_id
+group by
+	e.user_id,
+	e.course_id),
+course2 as (	
+	select
+		course_id,
+		COUNT(*) as lessons_in_course
+	from core.dim_lesson
+	group by 
+		course_id)
+select
+	au.user_id as user_id,
+	du.name as user_name,
+	au.course_id as course_id,
+	dc.title as course_title,
+	c2.lessons_in_course as lessons_in_course,
+	au.lessons_viewed as lessons_viewed,
+	ROUND(au.lessons_viewed * 1.0 / c2.lessons_in_course , 2) as completion_rate
+from
+	activity_users au
+left join core.dim_user du on au.user_id = du.user_id
+left join core.dim_course dc on au.course_id = dc.course_id
+left join course2 c2 on au.course_id = c2.course_id
+order by
+	au.user_id,
+	au.course_id;

APolikanov/2 - GreenPlum-Pet-project/IMPORT_DATA/courses_202512050730.csv

@@ -0,0 +1,4 @@
+"id","title","category","created_at"
+1,SQL для начинающих,data,2023-01-01
+2,Python для анализа данных,programming,2023-01-15
+3,BI с нуля,business,2023-03-01

APolikanov/2 - GreenPlum-Pet-project/IMPORT_DATA/enrollments_202512050730.csv

@@ -0,0 +1,6 @@
+"id","user_id","course_id","enrolled_at"
+1,1,1,2023-01-15
+2,1,2,2023-02-01
+3,2,1,2023-01-20
+4,3,2,2023-03-05
+5,4,3,2023-04-01

APolikanov/2 - GreenPlum-Pet-project/IMPORT_DATA/lesson_views_202512050730.csv

@@ -0,0 +1,7 @@
+"id","user_id","lesson_id","viewed_at"
+1,1,1,2023-01-16 10:00:00.000
+2,1,2,2023-01-16 10:15:00.000
+3,2,1,2023-01-21 09:00:00.000
+4,3,3,2023-03-06 12:00:00.000
+5,4,4,2023-04-02 10:00:00.000
+6,4,5,2023-04-03 10:00:00.000

APolikanov/2 - GreenPlum-Pet-project/IMPORT_DATA/lessons_202512050730.csv

@@ -0,0 +1,6 @@
+"id","course_id","title","duration_min"
+1,1,SELECT и FROM,10
+2,1,JOIN,15
+3,2,Pandas,20
+4,3,Основы BI,12
+5,3,Метрики и дашборды,18

APolikanov/2 - GreenPlum-Pet-project/IMPORT_DATA/users_202512050730.csv

@@ -0,0 +1,6 @@
+"id","name","age","email","registration_date"
+1,Alice,25,alice@mail.com,2023-01-10
+2,Bob,30,bob@gmail.com,2023-02-05
+3,Charlie,22,charlie@mail.com,2023-02-20
+4,Diana,28,diana@mail.com,2023-03-01
+5,Ethan,35,ethan@gmail.com,2023-03-10

agoncharov/clickhouse/greenplum/postgresql/create_table_insert_into.sql

@@ -0,0 +1,86 @@
+-- Этап 1: Создание таблиц и связей
+-- Создай все таблицы в SQL
+-- Заполни данными (INSERT)
+-- Настрой внешние ключи
+--Таблица 1: 
+   CREATE TABLE agoncharov.courses (
+	id int4 NOT NULL,
+	title varchar(50) NOT NULL,
+	category varchar(50) NOT NULL,
+	created_at date NOT NULL,
+	CONSTRAINT courses_pkey PRIMARY KEY (id)
+);	
+		insert into (id,title,category,created_at)
+		values (1,'SQL для начинающих','data','2023-01-01'),
+		(2,'Python для анализа данных','programming','2023-01-15'),
+		(3,'BI с нуля','business','2023-03-01');
+
+-- Таблица 2:
+CREATE TABLE agoncharov.enrollmets (
+	id int4 NOT NULL,
+	user_id int4 NULL,
+	course_id int4 NULL,
+	enrolled_at date NULL,
+	CONSTRAINT enrollmets_pkey PRIMARY KEY (id)
+);
+ALTER TABLE agoncharov.enrollmets ADD CONSTRAINT enrollmets_course_id_fkey FOREIGN KEY (course_id) REFERENCES agoncharov.courses(id);
+ALTER TABLE agoncharov.enrollmets ADD CONSTRAINT enrollmets_user_id_fkey FOREIGN KEY (user_id) REFERENCES agoncharov.users(id);
+
+INSERT INTO (id,user_id,course_id,enrolled_at)
+values (1,1,1,'2023-01-15'),(2,1,2,'2023-02-01'),(3,2,1,'2023-01-20'),
+(4,3,2,'2023-03-05'),(5,4,3,'2023-04-01');
+
+-- Таблица 3 :
+CREATE TABLE agoncharov.lesson_views (
+	id int4 NOT NULL,
+	user_id int4 NULL,
+	lesson_id int4 NULL,
+	viewed_at timestamp NULL,
+	CONSTRAINT lesson_views_pkey PRIMARY KEY (id)
+);
+
+
+-- agoncharov.lesson_views внешние включи
+
+ALTER TABLE agoncharov.lesson_views ADD CONSTRAINT lesson_views_lesson_id_fkey FOREIGN KEY (lesson_id) REFERENCES agoncharov.lessons(id);
+ALTER TABLE agoncharov.lesson_views ADD CONSTRAINT lesson_views_user_id_fkey FOREIGN KEY (user_id) REFERENCES agoncharov.users(id);
+
+insert into (id,user_id,lesson_id,viewed_at)
+values (1,1,1,'2023-01-16 10:00:00'),(2,1,2,'2023-01-16 10:15:00'),
+(3,2,1,'2023-01-21 09:00:00'), (4,3,3,'2023-04-06 12:00:00'),
+(5,4,4,'2023-04-02 10:00:00'),(6,4,5,'2023-04-03 10:00:00');
+
+-- Таблица 4
+CREATE TABLE agoncharov.lessons (
+	id int4 NOT NULL,
+	course_id int4 NULL,
+	title varchar(50) NOT NULL,
+	duration_min int4 NOT NULL,
+	CONSTRAINT lessons_pkey PRIMARY KEY (id)
+);
+
+
+-- agoncharov.lessons внешние включи
+
+ALTER TABLE agoncharov.lessons ADD CONSTRAINT lessons_course_id_fkey FOREIGN KEY (course_id) REFERENCES agoncharov.courses(id);
+
+insert into (id,course_id,title,duration_min)
+values (1,1,1,'SELECT и FROM',10),(2,2,1,'JOIN',15),
+(3,3,2,'PANDAS',20),(4,4,3,'Основы BI',12),(5,5,3,'Метрики и дашборды',18);
+
+-- Таблица 5:
+CREATE TABLE agoncharov.users (
+	id int4 NOT NULL,
+	"name" varchar(50) NOT NULL,
+	age int4 NULL,
+	email varchar(100) NULL,
+	registration_date date NULL,
+	CONSTRAINT users_pkey PRIMARY KEY (id)
+);
+
+INSERT INTO (id,name,age,email,registration_date)
+values (1,'Alice',25,'alice@mail.com','2023-01-10'),
+(2,'Bob',30,'bob@gmail.com','2023-02-05'),
+(3,'Charlie',22,'charlie@mail.com','2023-02-20'),
+(4,'Diana',28,'diana@mail.com','2023-03-01'),
+(5,'Ethan',35,'ethan@gmail.com','2023-03-10');

gmokrousov/postgresql/DM_course_summary.sql

@@ -0,0 +1,24 @@
+CREATE MATERIALIZED VIEW course_summary as 
+    with 
+        lw as (
+            select 
+                user_id,
+                lesson_id,
+                viewed_at 
+            from lesson_views
+        )
+    select
+        id as course_id,
+        title as course_title,
+        count(distinct e.user_id) as unique_users,
+        count(distinct l.lesson_id) as lessons_count,
+        count(distinct lw.viewed_at) as total_views,
+        round(count(distinct lw.viewed_at)::decimal / count(distinct e.user_id), 2) as avg_views_per_user,
+        min(lw.viewed_at) as first_viewed_at,
+        max(lw.viewed_at) as last_viewed_at 
+    from
+        courses as c 
+        join (select course_id, user_id from enrollments) as e on c.id = e.course_id 
+        join (select id as lesson_id, course_id from lessons) as l on c.id = l.course_id 
+        join lw on l.lesson_id = lw.lesson_id 
+    group by id;

gmokrousov/postgresql/DM_platform_summary.sql

@@ -0,0 +1,24 @@
+create materialized view platform_summary as 
+	with 
+        lv_stats as (
+            select 
+                count(distinct user_id) as users_with_views,
+                count(*)::decimal / nullif(count(distinct lesson_id), as avg_views_per_lesson
+            from lesson_views
+    	)
+	select 
+		total_users,
+		total_courses,
+		total_lessons,
+		users_with_views,
+		round((total_lessons::decimal / nullif(total_courses, 2)), 2) AS avg_lessons_per_course,
+		round(avg_views_per_lesson, 2) as avg_views_per_lesson
+	from 
+		(
+		select 
+			(select count(*) from users) as total_users,
+			(select count(*) from courses) as total_courses,
+			count(*) as total_lessons
+		from lessons
+	) as s
+	cross join lv_stats as lv;

gmokrousov/postgresql/DM_user_activity_summary.sql

@@ -0,0 +1,35 @@
+-- Витрина активности пользователей
+CREATE MATERIALIZED VIEW user_activity_summary as
+ with 
+    lessons_cte as (
+        select 
+            user_id,
+            count (*) as viewed_lessons_count,
+            max(viewed_at) as last_view,
+            min(viewed_at) as first_view
+        from lesson_views
+        group by user_id
+    ),
+    enrollments_cte as (
+        select user_id, count(*) as enrollment_count
+        from enrollments as e 
+        group by user_id
+    ),
+    engagement_cte as (
+        select user_id, count(*) as possible_lessons
+        from lessons as l
+        join enrollments as en on l.course_id = en.course_id
+        group by user_id
+    )   
+    select 
+        id as user_id, 
+        name, 
+        e.enrollment_count,
+        l.viewed_lessons_count,
+        l.last_view,
+        l.first_view,
+        round(l.viewed_lessons_count::decimal * 100 / nullif(en.possible_lessons, 0), 2) as engagement_rate
+    from users as u
+    join lessons_cte as l on u.id = l.user_id
+    join engagement_cte as en on u.id = en.user_id
+    join enrollments_cte as e on u.id = e.user_id;

gmokrousov/postgresql/creating_tables.sql

@@ -0,0 +1,69 @@
+create table users (
+    id serial primary key,
+    name varchar(50) not null,
+    age int not null,
+    email varchar(100) not null,
+    registration_date date not null
+);
+
+insert into users (name, age, email, registration_date) values
+    ('Alice', 25, 'alice@mail.com', ; '2023-01-10'),
+    ('Bob', 30, 'bob@gmail.com', '2023-02-05'),
+    ('Charlie', 22, 'Charlie@mail.com', '2023-02-20'),
+    ('Diana', 28, 'diana@mail.com', '2023-03-01'),
+    ('Ethan', 35, 'ethan@gmail.com', '2023-03-10');
+
+create table courses (
+    id serial primary key,
+    title varchar(100) not null,
+    category varchar(30) not null,
+    created_at date not_null
+);
+
+insert into courses (title, category, created_at) values
+    ('SQL для начинающих', 'data', '2023-01-01'),
+    ('Python для анализа данных', 'programming', '2023-01-15'),
+    ('BI с нуля', 'business', '2023-03-01');
+
+create table lessons (
+    id serial primary key,
+    course_id int references courses(id),
+    title varchar(100) not null,
+    duration_min int not null
+);
+
+insert into lessons (course_id, title, duration_min) values
+    (1, 'SELECT и FROM', 10),
+    (1, 'JOIN', 15),
+    (2, 'Pandas', 20),
+    (3, 'Основы BI', 12),
+    (3, 'Метрики и дашборды', 18);
+
+create table enrollments (
+    id serial primary key,
+    user_id int references users(id),
+    course_id int references courses(id),
+    enrolled_at date not null
+);
+
+insert into enrollments (user_id, course_id, enrolled_at) values
+    (1, 1, '2023-01-15'),
+    (1, 2, '2023-02-01'),
+    (2, 1, '2023-01-20'),
+    (3, 2, '2023-03-05'),
+    (4, 3, '2023-04-01');
+
+create table lesson_views (
+    id serial primary key,
+    user_id int references users(id),
+    lesson_id int references lessons(id),
+    viewed_at timestamp(0) not null
+);
+
+insert into lesson_views (user_id, lesson_id, viewed_at) values
+    (1, 1, '2023-01-16 10:00:00'),
+    (1, 2, '2023-01-16 10:15:00'),
+    (2, 1, '2023-01-21 09:00:00'),
+    (3, 3, '2023-03-06 12:00:00'),
+    (4, 4, '2023-04-02 10:00:00'),
+    (4, 5, '2023-04-03 10:00:00');

itikhomirov/clickhouse/Pet_project.sql

@@ -0,0 +1,418 @@
+-- Этап 1
+-- Задание 1
+-- Перенести данные как в пет-проекте с Greenplum
+
+CREATE TABLE itikhomirov.core_user
+ON CLUSTER clickhouse_cluster
+(
+    user_id Int32,
+    name String,
+    age Int32,
+    email String,
+    registration_date DateTime64(3, 'UTC')
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_user',
+    '{replica}'
+)
+ORDER BY user_id;
+
+ALTER TABLE itikhomirov.core_user
+MODIFY COLUMN registration_date DateTime64(3);
+
+CREATE TABLE itikhomirov.core_lesson 
+ON CLUSTER clickhouse_cluster
+(
+    lesson_id Int32,
+    title String,
+    duration_min Int32,
+    course_id Int32
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_lesson',
+    '{replica}'
+)
+ORDER BY lesson_id;
+
+CREATE TABLE itikhomirov.core_fact_lesson_views
+ON CLUSTER clickhouse_cluster
+(
+	user_id Int32, 
+    lesson_id Int32,
+    course_id Int32,
+    viewed_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_fact_lesson_views',
+    '{replica}'
+)
+ORDER BY user_id;
+
+ALTER TABLE itikhomirov.core_fact_lesson_views
+MODIFY COLUMN viewed_at DateTime64(3);
+
+DROP TABLE IF EXISTS itikhomirov.core_fact_lesson_views
+ON CLUSTER clickhouse_cluster;
+
+CREATE TABLE itikhomirov.core_enrollments
+ON CLUSTER clickhouse_cluster
+(
+	user_id Int32, 
+    course_id Int32,
+    enrolled_at DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_enrollmentsы',
+    '{replica}'
+)
+ORDER BY user_id;
+
+CREATE TABLE itikhomirov.core_course
+ON CLUSTER clickhouse_cluster
+(
+    course_id Int32,
+    title String,
+    category String
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/core_course',
+    '{replica}'
+)
+ORDER BY course_id;
+
+-- Этап 2. Создание витринных таблиц
+-- Задание 2: Подготовь структуру витрины lesson_popularity_summary
+
+-- Создаем пустую таблицу lesson_popularity_summary(структуру витрины)
+
+CREATE TABLE itikhomirov.lesson_popularity_summary
+ON CLUSTER clickhouse_cluster
+(
+    lesson_id     UInt32,
+    lesson_title  String,
+    course_id     UInt32,
+    course_title  String,
+    total_views   UInt32,
+    unique_users  UInt32,
+    first_view    DateTime64(3),
+    last_view     DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/lesson_popularity_summary',
+    '{replica}'
+)
+ORDER BY (lesson_id, course_id);
+
+-- Создаем Materialized View
+
+DROP TABLE IF EXISTS itikhomirov.mv_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_lesson_popularity_summary
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.lesson_popularity_summary
+AS
+SELECT
+    flv.lesson_id AS lesson_id,
+    l.title AS lesson_title,
+    flv.course_id AS course_id,
+    c.title AS course_title,
+    count() AS total_views,
+    uniqExact(flv.user_id) AS unique_users,
+    min(flv.viewed_at) AS first_view,
+    max(flv.viewed_at) AS last_view
+FROM itikhomirov.core_fact_lesson_views AS flv
+JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
+JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
+GROUP BY
+    flv.lesson_id,
+    l.title,
+    flv.course_id,
+    c.title;
+
+-- Вставляем данные в таблицу lesson_popularity_summary
+
+INSERT INTO itikhomirov.lesson_popularity_summary
+SELECT
+    flv.lesson_id,
+    l.title AS lesson_title,
+    flv.course_id,
+    c.title AS course_title,
+    count() AS total_views,
+    uniqExact(flv.user_id) AS unique_users,
+    min(flv.viewed_at) AS first_view,
+    max(flv.viewed_at) AS last_view
+FROM itikhomirov.core_fact_lesson_views AS flv
+JOIN itikhomirov.core_lesson  AS l ON flv.lesson_id = l.lesson_id
+JOIN itikhomirov.core_course  AS c ON flv.course_id = c.course_id
+GROUP BY
+    flv.lesson_id,
+    l.title,
+    flv.course_id,
+    c.title;
+
+-- Задание 3: Создай витрину inactive_users_summary
+
+-- Создаем пустую таблицу inactive_users_summary(структуру витрины)
+
+DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+DROP TABLE IF EXISTS itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE TABLE itikhomirov.inactive_users_summary
+ON CLUSTER clickhouse_cluster
+(
+    user_id UInt32,
+    name String,
+    email String,
+    age UInt8,
+    registered_courses_count UInt32,
+    registration_date DateTime64(3)
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/inactive_users_summary',
+    '{replica}'
+)
+ORDER BY (user_id);
+
+-- Создаем Materialized View
+
+DROP TABLE IF EXISTS itikhomirov.mv_inactive_users_summary
+ON CLUSTER clickhouse_cluster;
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_inactive_users_summary
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.inactive_users_summary
+AS
+WITH user_courses AS (
+    SELECT
+        e.user_id,
+        countDistinct(e.course_id) AS registered_courses_count
+    FROM itikhomirov.core_enrollments AS e
+    GROUP BY
+        e.user_id
+)
+SELECT
+    u.user_id                                AS user_id,
+    u.name                                   AS name,
+    u.email                                  AS email,
+    u.age                                    AS age,
+    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
+    u.registration_date                      AS registration_date
+FROM itikhomirov.core_user AS u
+LEFT JOIN user_courses AS uc
+    ON u.user_id = uc.user_id
+WHERE u.user_id NOT IN (
+    SELECT DISTINCT user_id
+    FROM itikhomirov.core_fact_lesson_views
+);
+
+
+INSERT INTO itikhomirov.inactive_users_summary
+WITH user_courses AS (
+    SELECT
+        e.user_id,
+        countDistinct(e.course_id) AS registered_courses_count
+    FROM itikhomirov.core_enrollments AS e
+    GROUP BY
+        e.user_id
+)
+-- В CTE user_courses считается число курсов по каждому пользователю независимо от просмотров
+-- Снаружи берутся пользователи без просмотров (NOT IN по core_fact_lesson_views), 
+-- и к ним подмешивается уже готовый registered_courses_count.
+SELECT
+    u.user_id AS user_id,
+    u.name AS name,
+    u.email AS email,
+    u.age AS age,
+    coalesce(uc.registered_courses_count, 0) AS registered_courses_count,
+    u.registration_date AS registration_date
+FROM itikhomirov.core_user AS u
+LEFT JOIN user_courses AS uc
+    ON u.user_id = uc.user_id
+WHERE u.user_id NOT IN (
+    SELECT DISTINCT user_id
+    FROM itikhomirov.core_fact_lesson_views
+)
+ORDER BY u.user_id;
+
+-- Задание 4: Создай витрину course_completion_rate
+
+CREATE TABLE itikhomirov.course_completion_rate
+ON CLUSTER clickhouse_cluster
+(
+    user_id          UInt32,
+    user_name        String,
+    course_id        UInt32,
+    course_title     String,
+    lessons_in_course UInt32,
+    lessons_viewed    UInt32,
+    completion_rate   Float32
+)
+ENGINE = ReplicatedMergeTree(
+    '/clickhouse/tables/{shard}/itikhomirov/course_completion_rate',
+    '{replica}'
+)
+ORDER BY (user_id, course_id);
+
+
+-- Создаем Materialized View
+
+CREATE MATERIALIZED VIEW itikhomirov.mv_course_completion_rate
+ON CLUSTER clickhouse_cluster
+TO itikhomirov.course_completion_rate
+AS
+WITH
+lessons_per_course AS (
+    SELECT
+        course_id,
+        countDistinct(lesson_id) AS lessons_in_course
+    FROM itikhomirov.core_lesson
+    GROUP BY course_id
+),
+views_per_user_course AS (
+    SELECT
+        user_id,
+        course_id,
+        countDistinct(lesson_id) AS lessons_viewed
+    FROM itikhomirov.core_fact_lesson_views
+    GROUP BY user_id, course_id
+)
+SELECT
+    u.user_id AS user_id,
+    u.name AS user_name,
+    c.course_id AS course_id,
+    c.title AS course_title,
+    lpc.lessons_in_course AS lessons_in_course,
+    coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
+    ROUND(
+        COALESCE(
+            NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
+            / NULLIF(lpc.lessons_in_course * 1.0, 0),
+            0
+        ),
+        2
+    ) AS completion_rate
+FROM itikhomirov.core_user AS u
+JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
+JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
+JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
+LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
+ORDER BY
+    u.user_id,
+    c.course_id;
+
+-- Вставляем данные в таблицу pet.course_completion_rate
+
+INSERT INTO itikhomirov.course_completion_rate
+-- Создаем две временные таблицы:
+-- 1) Сколько уроков в каждом курсе
+WITH
+lessons_per_course AS (
+    SELECT
+        course_id,
+        countDistinct(lesson_id) AS lessons_in_course
+    FROM itikhomirov.core_lesson
+    GROUP BY course_id
+),
+-- 2) Сколько уроков каждого курса посмотрел пользователь
+views_per_user_course AS (
+    SELECT
+        user_id,
+        course_id,
+        countDistinct(lesson_id) AS lessons_viewed
+    FROM itikhomirov.core_fact_lesson_views
+    GROUP BY user_id, course_id
+)
+-- Основной запрос
+SELECT
+    u.user_id AS user_id, 
+    u.name AS user_name,
+    c.course_id AS course_id,
+    c.title AS course_title,
+    lpc.lessons_in_course AS lessons_in_course,
+    coalesce(vpc.lessons_viewed, 0) AS lessons_viewed,
+    ROUND(
+        COALESCE(
+            NULLIF(coalesce(vpc.lessons_viewed, 0) * 1.0, 0)
+            / NULLIF(lpc.lessons_in_course * 1.0, 0),
+            0
+        ),
+        2
+    ) AS completion_rate
+FROM itikhomirov.core_user AS u
+JOIN itikhomirov.core_enrollments AS e ON u.user_id  = e.user_id
+JOIN itikhomirov.core_course AS c ON e.course_id = c.course_id
+JOIN lessons_per_course AS lpc ON c.course_id = lpc.course_id
+LEFT JOIN views_per_user_course AS vpc ON vpc.user_id   = u.user_id AND vpc.course_id = c.course_id
+ORDER BY
+    u.user_id,
+    c.course_id;
+
+-- Этап 3. Анализ и запросы
+
+-- Задание 5: Напиши SQL-запросы к витринам
+
+-- Уроки с хотя бы одним просмотром
+SELECT
+    lesson_title,
+    course_title,
+    total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE total_views > 0
+ORDER BY lesson_title ASC;
+
+-- Топ‑3 урока внутри одного курса
+SELECT
+    lesson_title,
+    course_title,
+    total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE course_title = 'SQL для начинающих'
+ORDER BY total_views DESC
+LIMIT 3;
+
+-- Пользователи, которые завершили курсы на 100%
+SELECT
+    user_id,
+    user_name,
+    course_title,
+    completion_rate
+FROM itikhomirov.course_completion_rate
+WHERE completion_rate = 1
+ORDER BY user_id;
+
+-- Этап 4. Тестирование и валидация
+
+-- Задание 6: Сравни данные с Greenplum**
+
+-- 1. Выбери 2–3 значения из таблиц `_summary_source` в Greenplum
+
+
+-- 2. Сравни с ClickHouse-результатами:
+--    - Совпадает ли `SUM`, `AVG`, `MIN`, `MAX`
+--    - Проверяй `NULL`, `0`, округления
+
+
+-- 3. Напиши мини-отчёт: **"где совпадает, где нет, почему"**
+
+SELECT
+    sum(total_views)                 AS sum_total_views,
+    avg(total_views)                 AS avg_total_views,
+    min(total_views)                 AS min_total_views,
+    max(total_views)                 AS max_total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE lesson_id IN (1, 3, 5);
+
+SELECT
+    sum(total_views)                 AS sum_total_views,
+    avg(total_views)                 AS avg_total_views,
+    min(total_views)                 AS min_total_views,
+    max(total_views)                 AS max_total_views
+FROM itikhomirov.lesson_popularity_summary
+WHERE lesson_id = 999;
+
+-- Отчёт в .doc файле
+

itikhomirov/greenplum/pet_gp_infra.sql

@@ -0,0 +1,241 @@
+-- Этап 1. Инициализация проекта
+-- Задание 1: Создай схемы
+
+CREATE SCHEMA raw;
+CREATE SCHEMA core;
+CREATE SCHEMA dm;
+
+-- Задание 2: Создай raw-таблицы
+
+CREATE TABLE raw.users (
+	id int,
+	name varchar(15),
+	age int,
+	email varchar(30),
+	registration_date DATE
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.courses (
+	id int,
+	title text,
+	category varchar(30),
+	created_at DATE
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.lessons (
+	id int,
+	course_id int,
+	title text,
+	duration_min int
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.enrollments (
+	id int,
+	user_id int,
+	course_id int,
+	enrolled_at DATE
+)
+DISTRIBUTED BY (id);
+
+CREATE TABLE raw.lesson_views (
+	id int,
+	user_id int,
+	lesson_id int,
+	viewed_at timestamp
+)
+DISTRIBUTED BY (id);
+
+-- Этап 2. Загрузка данных (эмуляция Airflow)
+-- Задание 3: Выгрузить данные из PostgreSQL и Загрузить CSV в Greenplum
+
+
+-- Экспортировал данные из PostgreSQL в CSV-файлы и загрузил их в Greenplum
+
+-- Этап 3. Слой Core — обогащённые данные
+-- Задание 4: Измерения (dimension tables)
+
+-- Измерение пользователей
+CREATE TABLE core.dim_user (
+    user_id INT PRIMARY KEY,
+    name TEXT,
+    age INT,
+    email TEXT,
+    registration_date DATE
+);
+
+-- Измерение курсов
+CREATE TABLE core.dim_course (
+    course_id INT PRIMARY KEY,
+    title TEXT,
+    category TEXT
+);
+
+-- Измерение уроков
+CREATE TABLE core.dim_lesson (
+    lesson_id INT PRIMARY KEY,
+    title TEXT,
+    duration_min INT,
+    course_id INT REFERENCES core.dim_course(course_id)
+);
+
+-- Опционально: измерение времени
+CREATE TABLE core.dim_date (
+    date DATE PRIMARY KEY,
+    year INT,
+    month INT,
+    day INT,
+    weekday TEXT
+);
+
+-- Задание 5: Факт (fact table)
+
+CREATE TABLE core.fact_lesson_views (
+    user_id INT REFERENCES core.dim_user(user_id),
+    lesson_id INT REFERENCES core.dim_lesson(lesson_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    viewed_at TIMESTAMP
+);
+
+CREATE TABLE core.enrollments (
+    user_id INT REFERENCES core.dim_user(user_id),
+    course_id INT REFERENCES core.dim_course(course_id),
+    enrolled_at TIMESTAMP
+);
+
+-- Задание 6: ETL переброса данных из raw в core
+
+INSERT INTO core.dim_user (user_id, name, age, email, registration_date) -- ok
+SELECT DISTINCT
+    id::INT,
+    name::TEXT,
+    age::INT,
+    email::TEXT,
+    registration_date::DATE
+FROM raw.users;
+
+INSERT INTO core.dim_lesson (lesson_id, title, duration_min, course_id) -- ok
+SELECT DISTINCT
+	id::INT,
+	title::TEXT,
+	duration_min::INT,
+	course_id::INT
+FROM raw.lessons;
+
+INSERT INTO core.fact_lesson_views (user_id, lesson_id, course_id, viewed_at) --заполнить из нескольких таблиц с помощью join
+SELECT DISTINCT
+	lv.user_id::INT,
+	lv.lesson_id::INT,
+	le.course_id::INT,
+	lv.viewed_at::TIMESTAMP
+FROM raw.lesson_views AS lv
+JOIN raw.lessons AS le ON lv.lesson_id = le.id;
+
+INSERT INTO core.enrollments (user_id, course_id, enrolled_at) -- ok
+SELECT DISTINCT 
+	user_id::INT,
+	course_id::INT,
+	enrolled_at::DATE
+FROM raw.enrollments;
+
+INSERT INTO core.dim_course (course_id, title, category) -- ok
+SELECT DISTINCT 
+	id::INT,
+	title::TEXT,
+	category::VARCHAR
+FROM raw.courses;
+
+INSERT INTO core.dim_date (date, year, month, day, weekday)
+SELECT DISTINCT
+	d::DATE,
+	EXTRACT(YEAR FROM d)::INT,
+	EXTRACT(MONTH FROM d)::INT,
+	EXTRACT(DAY FROM d)::INT,
+	TO_CHAR(d, 'Day')::TEXT
+FROM ( 
+	SELECT registration_date AS d FROM raw.users
+	UNION
+	SELECT enrolled_at FROM raw.enrollments
+	UNION
+	SELECT viewed_at FROM raw.lesson_views
+) all_dates
+WHERE d IS NOT NULL;
+
+-- Надо сделать, чтобы вместо номера месяца выводилось название месяца
+
+ALTER TABLE core.dim_date
+ALTER COLUMN month TYPE text;
+
+UPDATE core.dim_date 
+SET month = TO_CHAR(date, 'Month');
+
+-- Этап 4: Подготовка витрин в Greenplum
+-- Задание 7: Витрина dm.lesson_popularity_summary
+
+CREATE MATERIALIZED VIEW dm.lesson_popularity_summary AS
+SELECT 
+	dl.lesson_id,
+	dl.title AS lesson_title,
+	dl.course_id,
+	dc.title AS course_title,
+	COUNT(flv.user_id) AS total_views,
+	COUNT(DISTINCT flv.user_id) AS unique_users,
+	MIN(flv.viewed_at) AS first_view,
+	MAX(flv.viewed_at) AS last_view
+FROM core.fact_lesson_views AS flv
+JOIN core.dim_lesson AS dl ON flv.lesson_id = dl.lesson_id
+JOIN core.dim_course AS dc ON dl.course_id = dc.course_id
+GROUP BY dl.lesson_id, dl.title, dl.course_id, dc.title
+ORDER BY dl.lesson_id ASC;
+
+-- Задание 8: Витрина dm.inactive_users_summary
+
+CREATE MATERIALIZED VIEW dm.inactive_users_summary AS
+SELECT
+	du.user_id,
+	du.name AS user_name,
+	du.email,
+	du.age,
+	COUNT(e.course_id) AS enrollments_count,
+	du.registration_date
+FROM core.dim_user AS du
+LEFT JOIN core.enrollments AS e ON du.user_id = e.user_id
+LEFT JOIN core.fact_lesson_views AS flv ON du.user_id = flv.user_id
+WHERE flv.user_id IS NULL
+GROUP BY du.user_id, du.name, du.email, du.age, du.registration_date;
+
+-- Задание 9: Витрина dm.course_completion_rate
+
+CREATE MATERIALIZED VIEW dm.course_completion_rate AS
+SELECT 
+	du.user_id,
+	du.name AS user_name,
+	dc.course_id,
+	dc.title AS course_title,
+	COUNT(DISTINCT dl.lesson_id) AS lesson_in_course,
+	COUNT(DISTINCT flv.lesson_id) AS lessons_viewed,
+	ROUND(
+		COALESCE(
+			NULLIF(COUNT(DISTINCT flv.lesson_id)::NUMERIC,0) / NULLIF(COUNT(DISTINCT dl.lesson_id), 0),
+			0
+		) ,2
+	) AS completion_rate
+FROM core.dim_user AS du
+JOIN core.enrollments AS e ON du.user_id = e.user_id
+JOIN core.dim_course AS dc ON e.course_id= dc.course_id
+JOIN core.dim_lesson AS dl ON dc.course_id = dl.course_id
+LEFT JOIN core.fact_lesson_views AS flv ON du.user_id = flv.user_id AND dc.course_id = flv.course_id
+GROUP BY du.user_id, user_name, dc.course_id, course_title
+ORDER BY du.user_id ASC;
+
+-- 🧪 Этап 5. Проверка
+-- Задание 10: Проверь витрины
+
+SELECT * FROM dm.lesson_popularity_summary LIMIT 10;
+SELECT * FROM dm.inactive_users_summary LIMIT 10;
+SELECT * FROM dm.course_completion_rate ORDER BY user_id ASC LIMIT 10;
+
+
+

skonev/postgresql/create_tables.sql

@@ -0,0 +1,45 @@
+BEGIN;
+
+CREATE TABLE SKONEV.users(
+	id INT PRIMARY KEY, 
+	name VARCHAR(100), 
+	age INT, 
+	email VARCHAR(255) UNIQUE, 
+	registration_date DATE NOT NULL
+	);
+
+CREATE TABLE SKONEV.courses(
+	id INT PRIMARY KEY,
+	title TEXT NOT NULL,
+	category VARCHAR(255) CHECK (category IN ('data', 'programming', 'business')),
+	created_at DATE NOT NULL
+	);
+
+CREATE TABLE SKONEV.lessons(
+	id INT PRIMARY KEY, 
+	course_id INT NOT NULL,
+	FOREIGN KEY (course_id) REFERENCES courses(id), 
+	title TEXT NOT NULL, 
+	duration_min INT NOT NULL
+	);
+
+
+CREATE TABLE SKONEV.enrollments(
+	id INT PRIMARY KEY, 
+	user_id INT NOT NULL,
+	FOREIGN KEY (user_id) REFERENCES users(id), 
+	course_id INT NOT NULL,
+	FOREIGN KEY (course_id) REFERENCES courses(id), 
+	enrolled_at DATE NOT NULL
+	);
+
+CREATE TABLE SKONEV.lesson_views(
+	id INT PRIMARY KEY, 
+	user_id INT NOT NULL,
+	FOREIGN KEY (user_id) REFERENCES users(id), 
+	lesson_id INT NOT NULL,
+	FOREIGN KEY (lesson_id) REFERENCES lessons(id), 
+	viewed_at TIMESTAMP NOT NULL
+	);
+
+COMMIT;

skonev/postgresql/dm_course_summary.sql

@@ -0,0 +1,47 @@
+BEGIN;
+
+CREATE MATERIALIZED VIEW course_summary AS 
+WITH courses_users AS (
+SELECT 
+	c.id AS course_id,
+	c.title AS course_name,
+	COUNT(DISTINCT user_id) AS unique_users
+FROM courses c
+LEFT join enrollments e ON c.id = e.course_id 
+GROUP BY c.id, c.title
+), 
+courses_views AS (
+    SELECT 
+        l.course_id, 
+        COUNT(lv.id) AS total_views,
+        MAX(lv.viewed_at) AS last_viewed_at,
+        MIN(lv.viewed_at) AS first_viewed_at
+    FROM lessons l
+    JOIN lesson_views lv ON l.id = lv.lesson_id 
+    GROUP BY l.course_id
+),
+courses_lessons AS (
+	SELECT
+		course_id,
+		COUNT(DISTINCT id) AS lessons_count
+	FROM lessons
+	GROUP BY course_id
+)
+
+SELECT 
+	cu.course_id,
+	cu.course_name,
+	cu.unique_users,
+	cl.lessons_count,
+	cv.total_views,
+	ROUND(cv.total_views::DECIMAL / NULLIF(cu.unique_users, 0), 2) AS avg_views_per_user,
+	cv.first_viewed_at,
+	cv.last_viewed_at
+FROM courses_users cu
+LEFT JOIN courses_views cv USING(course_id)
+LEFT JOIN courses_lessons cl USING(course_id)
+ORDER BY cu.course_id;
+
+COMMIT;
+
+

skonev/postgresql/dm_platform_summary.sql

@@ -0,0 +1,20 @@
+BEGIN;
+
+CREATE MATERIALIZED VIEW platform_summary AS 
+SELECT
+    (SELECT COUNT(*) FROM users) AS total_users,
+    (SELECT COUNT(*) FROM courses) AS total_courses,
+    (SELECT COUNT(*) FROM lessons) AS total_lessons,
+    (SELECT COUNT(DISTINCT user_id) FROM lesson_views) AS users_with_views,
+    ROUND(
+        (SELECT COUNT(*)::DECIMAL FROM lessons) / 
+        NULLIF((SELECT COUNT(DISTINCT course_id) FROM lessons), 0), 
+        2
+    ) AS avg_lessons_per_course,
+    ROUND(
+        (SELECT COUNT(*)::DECIMAL FROM lesson_views) / 
+        NULLIF((SELECT COUNT(DISTINCT lesson_id) FROM lesson_views), 0), 
+        2
+    ) AS avg_views_per_lesson;
+
+COMMIT;

skonev/postgresql/dm_user_activity_summary.sql

@@ -0,0 +1,38 @@
+BEGIN;
+
+CREATE MATERIALIZED VIEW user_activity_summary AS 
+WITH users_views AS (
+SELECT
+	u.id AS user_id,
+	u.name, 
+	COUNT(DISTINCT lv.lesson_id) AS viewed_lessons_count,
+	MAX(lv.viewed_at) AS last_view,
+	MIN(lv.viewed_at) AS first_view
+FROM users AS u
+JOIN lesson_views AS lv on u.id = lv.user_id
+GROUP BY u.id
+
+),
+users_engagement AS (
+SELECT 
+	e.user_id AS user_id,
+	COUNT(DISTINCT l.id) AS lessons_cnt,
+	COUNT(DISTINCT course_id) AS enrollment_count
+FROM enrollments AS e
+JOIN lessons AS l using(course_id)
+GROUP BY e.user_id
+)
+
+SELECT 
+	uv.user_id,
+	uv.name,
+	ue.enrollment_count,
+	uv.viewed_lessons_count,
+	uv.last_view,
+	uv.first_view,
+	ROUND(uv.viewed_lessons_count::DECIMAL / NULLIF(ue.lessons_cnt, 0) * 100, 2)  AS engagement_rate
+FROM users_views AS uv
+LEFT JOIN users_engagement AS ue using (user_id)
+ORDER BY uv.user_id
+
+COMMIT;

skonev/postgresql/insert_data.sql

@@ -0,0 +1,42 @@
+BEGIN;
+
+INSERT INTO SKONEV.users (id, name, age, email, registration_date)
+VALUES
+(1, 'Alice', 25, 'alice@mail.com', '2023-01-10'),
+(2, 'Bob', 30, 'bob@gmail.com', '2023-02-05'),
+(3, 'Charlie', 22, 'charlie@mail.com', '2023-02-20'),
+(4, 'Diana', 28, 'diana@mail.com', '2023-03-01'),
+(5, 'Ethan', 35, 'ethan@gmail.com', '2023-03-10');
+
+INSERT INTO SKONEV.courses (id, title, category, created_at)
+VALUES
+(1, 'SQL для начинающих', 'data', '2023-01-01'),
+(2, 'Python для анализа данных', 'programming', '2023-01-15'),
+(3, 'BI с нуля', 'business', '2023-03-01');
+
+INSERT INTO SKONEV.lessons (id, course_id, title, duration_min)
+VALUES
+(1, 1, 'SELECT и FROM', 10),
+(2, 1, 'JOIN', 15),
+(3, 2, 'Pandas', 20),
+(4, 3, 'Основы BI', 12),
+(5, 3, 'Метрики и дашборды', 18);
+
+INSERT INTO SKONEV.enrollments (id, user_id, course_id, enrolled_at)
+VALUES
+(1, 1, 1, '2023-01-15'),
+(2, 1, 2, '2023-02-01'),
+(3, 2, 1, '2023-01-20'),
+(4, 3, 2, '2023-03-05'),
+(5, 4, 3, '2023-04-01');
+
+INSERT INTO SKONEV.lesson_views (id, user_id, lesson_id, viewed_at)
+VALUES
+(1, 1, 1, '2023-01-16 10:00:00'),
+(2, 1, 2, '2023-01-16 10:15:00'),
+(3, 2, 1, '2023-01-21 09:00:00'),
+(4, 3, 3, '2023-03-06 12:00:00'),
+(5, 4, 4, '2023-04-02 10:00:00'),
+(6, 4, 5, '2023-04-03 10:00:00');
+
+COMMIT;

test.txt

@@ -0,0 +1 @@
+test коммита в Gitea