From b625abae688742bffcd096e822f62027dc182170 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Fri, 8 Dec 2023 20:49:01 +0800 Subject: [PATCH 01/19] change docker compose file --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 13285d2..ef5ca27 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: - ./logs:/app/logs environment: - BASE_URL=http://localhost:5173 - - MONGO_URL=mongodb://db:27017/odyssey-test + - MONGO_URL=mongodb://db:27017/linux-odyssey - DOCKER_NETWORK=linux-odyssey-players - SECRET_KEY=b1d2e917-17b9-4fce-9055-082dee6ec5d6 # Only for testing - TRUSTED_PROXIES=${TRUSTED_PROXIES} @@ -64,7 +64,7 @@ services: ports: - '3001:3001' environment: - - MONGO_URL=mongodb://db:27017/odyssey-test + - MONGO_URL=mongodb://db:27017/linux-odyssey - ADMIN_USERNAME=alex - ADMIN_PASSWORD=Alex1234 volumes: From c243e29002a46c5e53f416deb8e806f594491478 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Sun, 10 Dec 2023 01:25:27 +0800 Subject: [PATCH 02/19] fix: re-calculate completed rate in analytics --- packages/analytics/src/controllers/index.js | 8 ++++---- packages/analytics/src/models/profiles.js | 20 ++++++++++++++++++++ packages/analytics/views/home.handlebars | 8 ++++---- 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 packages/analytics/src/models/profiles.js diff --git a/packages/analytics/src/controllers/index.js b/packages/analytics/src/controllers/index.js index d5a6967..aa9980f 100644 --- a/packages/analytics/src/controllers/index.js +++ b/packages/analytics/src/controllers/index.js @@ -1,15 +1,15 @@ import { userCount } from '../models/users.js' -import { sessionCount, sessionCompleted } from '../models/sessions.js' +import { questCount, questCompleted } from '../models/profiles.js' // eslint-disable-next-line import/prefer-default-export export async function homeController(req, res) { const payload = { title: 'Linux Odyssey Admin', userCount: await userCount(), - sessionCount: await sessionCount(), - sessionCompleted: await sessionCompleted(), + questCount: await questCount(), + questCompleted: await questCompleted(), } payload.completedRate = - Math.round((payload.sessionCompleted / payload.sessionCount) * 10000) / 100 + Math.round((payload.questCompleted / payload.questCount) * 10000) / 100 res.render('home', payload) } diff --git a/packages/analytics/src/models/profiles.js b/packages/analytics/src/models/profiles.js new file mode 100644 index 0000000..669afff --- /dev/null +++ b/packages/analytics/src/models/profiles.js @@ -0,0 +1,20 @@ +/* eslint-disable object-shorthand */ +/* eslint-disable func-names */ +import { UserProfile } from '@linux-odyssey/models' + +export function questCount() { + return UserProfile.find().count() +} + +export function questCompleted() { + return UserProfile.find({ + $where: function () { + for (const key in this.progress) { + if (this.progress[key].completed === true) { + return true + } + } + return false + }, + }).count() +} diff --git a/packages/analytics/views/home.handlebars b/packages/analytics/views/home.handlebars index cd8148a..3098257 100644 --- a/packages/analytics/views/home.handlebars +++ b/packages/analytics/views/home.handlebars @@ -7,12 +7,12 @@ {{userCount}} - Session Count - {{sessionCount}} + Total Started Quests + {{questCount}} - Completed Session - {{sessionCompleted}} + Completed Quests + {{questCompleted}} Completed Rate From 9200336b499173bfe2472dd1c17fb688a2ad81c4 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Sun, 10 Dec 2023 02:21:02 +0800 Subject: [PATCH 03/19] fix: analytics modify --- packages/analytics/src/controllers/index.js | 8 ++++---- packages/analytics/src/models/profiles.js | 8 ++++++-- packages/analytics/views/home.handlebars | 6 +++--- packages/analytics/views/sessionDetail.handlebars | 10 +++++----- packages/analytics/views/sessions.handlebars | 4 ++-- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/analytics/src/controllers/index.js b/packages/analytics/src/controllers/index.js index aa9980f..77e736b 100644 --- a/packages/analytics/src/controllers/index.js +++ b/packages/analytics/src/controllers/index.js @@ -1,15 +1,15 @@ import { userCount } from '../models/users.js' -import { questCount, questCompleted } from '../models/profiles.js' +import { totalQuests, totalCompleted } from '../models/profiles.js' // eslint-disable-next-line import/prefer-default-export export async function homeController(req, res) { const payload = { title: 'Linux Odyssey Admin', userCount: await userCount(), - questCount: await questCount(), - questCompleted: await questCompleted(), + totalQuests: await totalQuests(), + totalCompleted: await totalCompleted(), } payload.completedRate = - Math.round((payload.questCompleted / payload.questCount) * 10000) / 100 + Math.round((payload.totalCompleted / payload.totalQuests) * 10000) / 100 res.render('home', payload) } diff --git a/packages/analytics/src/models/profiles.js b/packages/analytics/src/models/profiles.js index 669afff..31cbb67 100644 --- a/packages/analytics/src/models/profiles.js +++ b/packages/analytics/src/models/profiles.js @@ -2,11 +2,11 @@ /* eslint-disable func-names */ import { UserProfile } from '@linux-odyssey/models' -export function questCount() { +export function totalQuests() { return UserProfile.find().count() } -export function questCompleted() { +export function totalCompleted() { return UserProfile.find({ $where: function () { for (const key in this.progress) { @@ -18,3 +18,7 @@ export function questCompleted() { }, }).count() } + +export function userProfileList() { + return UserProfile.find() +} diff --git a/packages/analytics/views/home.handlebars b/packages/analytics/views/home.handlebars index 3098257..d96f890 100644 --- a/packages/analytics/views/home.handlebars +++ b/packages/analytics/views/home.handlebars @@ -1,6 +1,6 @@

Welcome to {{title}}

-

Users

+

Overview

@@ -8,11 +8,11 @@ - + - + diff --git a/packages/analytics/views/sessionDetail.handlebars b/packages/analytics/views/sessionDetail.handlebars index d43b8fe..c209bfc 100644 --- a/packages/analytics/views/sessionDetail.handlebars +++ b/packages/analytics/views/sessionDetail.handlebars @@ -36,20 +36,20 @@
Registered Users
Total Started Quests{{questCount}}{{totalQuests}}
Completed Quests{{questCompleted}}{{totalCompleted}}
Completed Rate
- + + - - + {{#each commands}} - + + - {{/each}} diff --git a/packages/analytics/views/sessions.handlebars b/packages/analytics/views/sessions.handlebars index 6fff971..99469bb 100644 --- a/packages/analytics/views/sessions.handlebars +++ b/packages/analytics/views/sessions.handlebars @@ -4,23 +4,23 @@ - + {{#each sessions}} - + {{/each}}
CommandStage PWDInput Output ErrorStageCreated AtInput Time
{{command}}{{stage}} {{pwd}}{{command}} {{output}} {{error}}{{stage}} {{createdAt}}
User Quest Status Created At Finished At Used Time Command CountSession
{{user}} {{quest}}Detail {{status}} {{createdAt}} {{finishedAt}} {{usedTime}} {{commandCount}}Detail
From ac31ca8659764b4d50e92444c0a572c6b3081267 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Sun, 10 Dec 2023 03:21:30 +0800 Subject: [PATCH 04/19] feat: show analytics data page by page --- .../analytics/src/controllers/commands.js | 26 ++++++++++++++++--- .../analytics/src/controllers/sessions.js | 19 +++++++++++--- packages/analytics/src/controllers/users.js | 20 +++++++++++--- packages/analytics/src/models/commands.js | 6 ++++- packages/analytics/src/models/sessions.js | 6 ++++- packages/analytics/src/models/users.js | 7 ++++- packages/analytics/views/commands.handlebars | 5 ++++ packages/analytics/views/sessions.handlebars | 5 ++++ packages/analytics/views/users.handlebars | 5 ++++ 9 files changed, 87 insertions(+), 12 deletions(-) diff --git a/packages/analytics/src/controllers/commands.js b/packages/analytics/src/controllers/commands.js index 4692652..514a891 100644 --- a/packages/analytics/src/controllers/commands.js +++ b/packages/analytics/src/controllers/commands.js @@ -1,9 +1,29 @@ +/* eslint-disable object-shorthand */ +import { Command } from '@linux-odyssey/models' import { errorCommands } from '../models/commands.js' // eslint-disable-next-line import/prefer-default-export export async function commandListController(req, res) { - const payload = { - errorCommands: await errorCommands(), + const itemsPerPage = 250 + + try { + const totalCommands = await Command.countDocuments({ + $and: [ + { error: { $exists: true } }, + { error: { $ne: null } }, + { error: { $ne: '' } }, + ], + }) + + const maxPages = Math.ceil(totalCommands / itemsPerPage) + + const pageNumber = req.query.page || 1 + const payload = { + errorCommands: await errorCommands(pageNumber, itemsPerPage), + pages: Array.from({ length: maxPages }, (_, index) => index + 1), + } + res.render('commands', payload) + } catch (error) { + res.status(500).send('Error fetching data') } - res.render('commands', payload) } diff --git a/packages/analytics/src/controllers/sessions.js b/packages/analytics/src/controllers/sessions.js index 47dc3bb..e0b131b 100644 --- a/packages/analytics/src/controllers/sessions.js +++ b/packages/analytics/src/controllers/sessions.js @@ -1,8 +1,21 @@ -import { sessionList, sessionDetail } from '../models/sessions.js' +import { sessionCount, sessionList, sessionDetail } from '../models/sessions.js' export async function sessionListController(req, res) { - const sessions = await sessionList() - res.render('sessions', { sessions }) + const pageNumber = req.query.page || 1 + const itemsPerPage = 100 + + try { + const totalSessions = await sessionCount() + const maxPages = Math.ceil(totalSessions / itemsPerPage) + + const sessions = await sessionList(pageNumber, itemsPerPage) + + const pages = Array.from({ length: maxPages }, (_, index) => index + 1) + + res.render('sessions', { sessions, pages }) + } catch (error) { + res.status(500).send('Error fetching session data') + } } export async function sessionDetailController(req, res) { diff --git a/packages/analytics/src/controllers/users.js b/packages/analytics/src/controllers/users.js index 5eb16b9..828a0c8 100644 --- a/packages/analytics/src/controllers/users.js +++ b/packages/analytics/src/controllers/users.js @@ -1,6 +1,20 @@ -import { userList } from '../models/users.js' +import { userList, userCount } from '../models/users.js' -// eslint-disable-next-line import/prefer-default-export export async function userListController(req, res) { - res.render('users', { users: await userList() }) + const pageNumber = req.query.page || 1 + const itemsPerPage = 50 + + try { + const totalUsers = await userCount() + const maxPages = Math.ceil(totalUsers / itemsPerPage) + + const users = await userList(pageNumber, itemsPerPage) + + res.render('users', { + users, + maxPages: Array.from({ length: maxPages }, (_, index) => index + 1), + }) + } catch (error) { + res.status(500).send('Error fetching user data') + } } diff --git a/packages/analytics/src/models/commands.js b/packages/analytics/src/models/commands.js index 48ece73..c2a756a 100644 --- a/packages/analytics/src/models/commands.js +++ b/packages/analytics/src/models/commands.js @@ -1,7 +1,9 @@ import { Command } from '@linux-odyssey/models' // eslint-disable-next-line import/prefer-default-export -export async function errorCommands() { +export async function errorCommands(pageNumber, itemsPerPage) { + const skipAmount = (pageNumber - 1) * itemsPerPage + const commands = await Command.find({ $and: [ { error: { $exists: true } }, @@ -13,6 +15,8 @@ export async function errorCommands() { path: 'session', populate: { path: 'user' }, }) + .skip(skipAmount) + .limit(itemsPerPage) .exec() return commands.map(({ command, error, createdAt, session }) => { const { user, quest } = session diff --git a/packages/analytics/src/models/sessions.js b/packages/analytics/src/models/sessions.js index 72d5aa4..a5568d8 100644 --- a/packages/analytics/src/models/sessions.js +++ b/packages/analytics/src/models/sessions.js @@ -30,7 +30,9 @@ function formatTime(time) { return `${hours}h ${minutes % 60}m ${seconds % 60}s` } -export async function sessionList() { +export async function sessionList(pageNumber, itemsPerPage) { + const skipAmount = (pageNumber - 1) * itemsPerPage + const sessions = await Session.aggregate([ { $lookup: { @@ -50,6 +52,8 @@ export async function sessionList() { }, }, ]) + .skip(skipAmount) + .limit(itemsPerPage) return sessions.map( ({ _id, diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index 69d8310..eb501e1 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -11,7 +11,9 @@ export function userCount() { return User.find().count() } -export async function userList() { +export async function userList(pageNumber, itemsPerPage) { + const skipAmount = (pageNumber - 1) * itemsPerPage + const users = await Session.aggregate([ { $group: { @@ -30,7 +32,10 @@ export async function userList() { }, { $unwind: '$userData' }, { $sort: { 'userData.createdAt': 1 } }, + { $skip: skipAmount }, + { $limit: itemsPerPage }, ]) + return users.map(({ userData, count, lastActivityAt }) => { const { username, email, createdAt } = userData return { diff --git a/packages/analytics/views/commands.handlebars b/packages/analytics/views/commands.handlebars index 16f23c2..00843c9 100644 --- a/packages/analytics/views/commands.handlebars +++ b/packages/analytics/views/commands.handlebars @@ -21,4 +21,9 @@ {{/each}} +
+ \ No newline at end of file diff --git a/packages/analytics/views/sessions.handlebars b/packages/analytics/views/sessions.handlebars index 99469bb..a9abf8c 100644 --- a/packages/analytics/views/sessions.handlebars +++ b/packages/analytics/views/sessions.handlebars @@ -24,4 +24,9 @@ {{/each}} + + \ No newline at end of file diff --git a/packages/analytics/views/users.handlebars b/packages/analytics/views/users.handlebars index acc6233..c2b73f2 100644 --- a/packages/analytics/views/users.handlebars +++ b/packages/analytics/views/users.handlebars @@ -20,4 +20,9 @@ {{/each}} + + \ No newline at end of file From a578d4f9100f0973568c8a3a18896776195aae1d Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Mon, 11 Dec 2023 01:09:06 +0800 Subject: [PATCH 05/19] fix: analytics completed rate --- packages/analytics/src/models/profiles.js | 58 +++++++++++++++++++---- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/packages/analytics/src/models/profiles.js b/packages/analytics/src/models/profiles.js index 31cbb67..cb95ff1 100644 --- a/packages/analytics/src/models/profiles.js +++ b/packages/analytics/src/models/profiles.js @@ -3,20 +3,58 @@ import { UserProfile } from '@linux-odyssey/models' export function totalQuests() { - return UserProfile.find().count() + return UserProfile.aggregate([ + { + $project: { + _id: 0, + progress: { $objectToArray: '$progress' }, + }, + }, + { + $unwind: '$progress', + }, + { + $group: { + _id: null, + totalCount: { $sum: 1 }, + }, + }, + ]).then((results) => { + if (results.length === 0) { + throw new Error('No documents found') + } + return results[0].totalCount + }) } export function totalCompleted() { - return UserProfile.find({ - $where: function () { - for (const key in this.progress) { - if (this.progress[key].completed === true) { - return true - } - } - return false + return UserProfile.aggregate([ + { + $project: { + _id: 0, + progress: { $objectToArray: '$progress' }, + }, }, - }).count() + { + $unwind: '$progress', + }, + { + $match: { + 'progress.v.completed': true, + }, + }, + { + $group: { + _id: null, + totalCount: { $sum: 1 }, + }, + }, + ]).then((results) => { + if (results.length === 0) { + throw new Error('No documents found') + } + return results[0].totalCount + }) } export function userProfileList() { From 61f627b1259038d1c50ccd8d95e08596f8962711 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Mon, 11 Dec 2023 17:55:15 +0800 Subject: [PATCH 06/19] feat: add quest page in analytics --- packages/analytics/src/controllers/index.js | 2 +- packages/analytics/src/controllers/quests.js | 7 ++ packages/analytics/src/models/profiles.js | 62 ---------- packages/analytics/src/models/quests.js | 114 ++++++++++++++++++ packages/analytics/src/router.js | 2 + .../analytics/views/layouts/main.handlebars | 1 + packages/analytics/views/quests.handlebars | 18 +++ .../analytics/views/userDetail.handlebars | 0 packages/analytics/views/users.handlebars | 18 +-- 9 files changed, 152 insertions(+), 72 deletions(-) create mode 100644 packages/analytics/src/controllers/quests.js delete mode 100644 packages/analytics/src/models/profiles.js create mode 100644 packages/analytics/src/models/quests.js create mode 100644 packages/analytics/views/quests.handlebars create mode 100644 packages/analytics/views/userDetail.handlebars diff --git a/packages/analytics/src/controllers/index.js b/packages/analytics/src/controllers/index.js index 77e736b..8557cba 100644 --- a/packages/analytics/src/controllers/index.js +++ b/packages/analytics/src/controllers/index.js @@ -1,5 +1,5 @@ import { userCount } from '../models/users.js' -import { totalQuests, totalCompleted } from '../models/profiles.js' +import { totalQuests, totalCompleted } from '../models/quests.js' // eslint-disable-next-line import/prefer-default-export export async function homeController(req, res) { diff --git a/packages/analytics/src/controllers/quests.js b/packages/analytics/src/controllers/quests.js new file mode 100644 index 0000000..e11dc39 --- /dev/null +++ b/packages/analytics/src/controllers/quests.js @@ -0,0 +1,7 @@ +import { questList } from '../models/quests.js' + +// eslint-disable-next-line import/prefer-default-export +export async function questListController(req, res) { + const quests = await questList() + res.render('quests', { quests }) +} diff --git a/packages/analytics/src/models/profiles.js b/packages/analytics/src/models/profiles.js deleted file mode 100644 index cb95ff1..0000000 --- a/packages/analytics/src/models/profiles.js +++ /dev/null @@ -1,62 +0,0 @@ -/* eslint-disable object-shorthand */ -/* eslint-disable func-names */ -import { UserProfile } from '@linux-odyssey/models' - -export function totalQuests() { - return UserProfile.aggregate([ - { - $project: { - _id: 0, - progress: { $objectToArray: '$progress' }, - }, - }, - { - $unwind: '$progress', - }, - { - $group: { - _id: null, - totalCount: { $sum: 1 }, - }, - }, - ]).then((results) => { - if (results.length === 0) { - throw new Error('No documents found') - } - return results[0].totalCount - }) -} - -export function totalCompleted() { - return UserProfile.aggregate([ - { - $project: { - _id: 0, - progress: { $objectToArray: '$progress' }, - }, - }, - { - $unwind: '$progress', - }, - { - $match: { - 'progress.v.completed': true, - }, - }, - { - $group: { - _id: null, - totalCount: { $sum: 1 }, - }, - }, - ]).then((results) => { - if (results.length === 0) { - throw new Error('No documents found') - } - return results[0].totalCount - }) -} - -export function userProfileList() { - return UserProfile.find() -} diff --git a/packages/analytics/src/models/quests.js b/packages/analytics/src/models/quests.js new file mode 100644 index 0000000..9acb2cb --- /dev/null +++ b/packages/analytics/src/models/quests.js @@ -0,0 +1,114 @@ +/* eslint-disable object-shorthand */ +/* eslint-disable func-names */ +import { UserProfile, Quest } from '@linux-odyssey/models' + +export function totalQuests() { + return UserProfile.aggregate([ + { + $project: { + _id: 0, + progress: { $objectToArray: '$progress' }, + }, + }, + { + $unwind: '$progress', + }, + { + $group: { + _id: null, + totalCount: { $sum: 1 }, + }, + }, + ]).then((results) => { + if (results.length === 0) { + throw new Error('No documents found') + } + return results[0].totalCount + }) +} + +export function totalCompleted() { + return UserProfile.aggregate([ + { + $project: { + _id: 0, + progress: { $objectToArray: '$progress' }, + }, + }, + { + $unwind: '$progress', + }, + { + $match: { + 'progress.v.completed': true, + }, + }, + { + $group: { + _id: null, + totalCount: { $sum: 1 }, + }, + }, + ]).then((results) => { + if (results.length === 0) { + throw new Error('No documents found') + } + return results[0].totalCount + }) +} + +export function questList() { + return UserProfile.aggregate([ + { + $project: { + progress: { + $objectToArray: '$progress', + }, + }, + }, + { + $unwind: '$progress', + }, + { + $group: { + _id: '$progress.v.quest', + started: { + $sum: 1, + }, + completed: { + $sum: { + $cond: { + if: { $eq: ['$progress.v.completed', true] }, + then: 1, + else: 0, + }, + }, + }, + }, + }, + { + $project: { + _id: 1, + started: 1, + completed: 1, + completedRate: { + $cond: { + if: { $eq: ['$started', 0] }, + then: 0, + else: { + $round: [ + { $multiply: [{ $divide: ['$completed', '$started'] }, 100] }, + 2, + ], + }, + }, + }, + }, + }, + { + $sort: { + _id: 1, // 按任务ID升序排序 + }, + }, + ]) +} diff --git a/packages/analytics/src/router.js b/packages/analytics/src/router.js index f5a458b..9ef6ba6 100644 --- a/packages/analytics/src/router.js +++ b/packages/analytics/src/router.js @@ -2,6 +2,7 @@ import { Router } from 'express' import { homeController } from './controllers/index.js' import { userListController } from './controllers/users.js' +import { questListController } from './controllers/quests.js' import { commandListController } from './controllers/commands.js' import { sessionDetailController, @@ -12,6 +13,7 @@ const router = Router() router.get('/', homeController) router.get('/users', userListController) +router.get('/quests', questListController) router.get('/commands', commandListController) router.get('/sessions', sessionListController) router.get('/sessions/:id', sessionDetailController) diff --git a/packages/analytics/views/layouts/main.handlebars b/packages/analytics/views/layouts/main.handlebars index e269d7b..6306555 100644 --- a/packages/analytics/views/layouts/main.handlebars +++ b/packages/analytics/views/layouts/main.handlebars @@ -8,6 +8,7 @@ diff --git a/packages/analytics/views/quests.handlebars b/packages/analytics/views/quests.handlebars new file mode 100644 index 0000000..6107a76 --- /dev/null +++ b/packages/analytics/views/quests.handlebars @@ -0,0 +1,18 @@ +

Quests

+
+ + + + + + + + {{#each quests}} + + + + + + + {{/each}} +
IdStartedCompletedCompleted Rate
{{_id}}{{started}}{{completed}}{{completedRate}}
\ No newline at end of file diff --git a/packages/analytics/views/userDetail.handlebars b/packages/analytics/views/userDetail.handlebars new file mode 100644 index 0000000..e69de29 diff --git a/packages/analytics/views/users.handlebars b/packages/analytics/views/users.handlebars index c2b73f2..fcbedf7 100644 --- a/packages/analytics/views/users.handlebars +++ b/packages/analytics/views/users.handlebars @@ -3,20 +3,20 @@ - - - - + + + + {{#each users}} - - - - - + + + + + {{/each}}
UsernameEmailRegistration MethodRegistration TimeSessions Count Last ActivitySessions CountEmailRegistration TimeRegistration Method
{{username}}{{email}}{{method}}{{createdAt}}{{sessionCount}}{{username}} {{lastActivityAt}}{{sessionCount}}{{email}}{{createdAt}}{{method}}
From 3e7092dbd63cd2bb1ba9671bc7384abc564e36a1 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Mon, 11 Dec 2023 21:13:06 +0800 Subject: [PATCH 07/19] feat: modify analytics --- packages/analytics/src/controllers/quests.js | 3 +++ packages/analytics/src/models/quests.js | 6 +++--- packages/analytics/views/quests.handlebars | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/analytics/src/controllers/quests.js b/packages/analytics/src/controllers/quests.js index e11dc39..4e5b12a 100644 --- a/packages/analytics/src/controllers/quests.js +++ b/packages/analytics/src/controllers/quests.js @@ -3,5 +3,8 @@ import { questList } from '../models/quests.js' // eslint-disable-next-line import/prefer-default-export export async function questListController(req, res) { const quests = await questList() + quests.forEach((quest) => { + quest.failed = quest.started - quest.completed + }) res.render('quests', { quests }) } diff --git a/packages/analytics/src/models/quests.js b/packages/analytics/src/models/quests.js index 9acb2cb..b5f7471 100644 --- a/packages/analytics/src/models/quests.js +++ b/packages/analytics/src/models/quests.js @@ -1,6 +1,6 @@ /* eslint-disable object-shorthand */ /* eslint-disable func-names */ -import { UserProfile, Quest } from '@linux-odyssey/models' +import { UserProfile } from '@linux-odyssey/models' export function totalQuests() { return UserProfile.aggregate([ @@ -98,7 +98,7 @@ export function questList() { else: { $round: [ { $multiply: [{ $divide: ['$completed', '$started'] }, 100] }, - 2, + 0, ], }, }, @@ -107,7 +107,7 @@ export function questList() { }, { $sort: { - _id: 1, // 按任务ID升序排序 + completed: -1, }, }, ]) diff --git a/packages/analytics/views/quests.handlebars b/packages/analytics/views/quests.handlebars index 6107a76..9347e7e 100644 --- a/packages/analytics/views/quests.handlebars +++ b/packages/analytics/views/quests.handlebars @@ -5,6 +5,7 @@ Id Started Completed + Failed Completed Rate {{#each quests}} @@ -12,7 +13,8 @@ {{_id}} {{started}} {{completed}} - {{completedRate}} + {{failed}} + {{completedRate}} % {{/each}} \ No newline at end of file From 1b94ae75ca5d2bc9f8ab5107645a5db39d2400a8 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Wed, 13 Dec 2023 14:40:47 +0800 Subject: [PATCH 08/19] feat: change to nextKey in users --- packages/analytics/src/controllers/users.js | 13 ++++++++----- packages/analytics/src/models/users.js | 17 +++++++++++------ packages/analytics/views/users.handlebars | 4 +--- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/analytics/src/controllers/users.js b/packages/analytics/src/controllers/users.js index 828a0c8..55894d1 100644 --- a/packages/analytics/src/controllers/users.js +++ b/packages/analytics/src/controllers/users.js @@ -1,20 +1,23 @@ import { userList, userCount } from '../models/users.js' export async function userListController(req, res) { - const pageNumber = req.query.page || 1 + const { nextKey } = req.query const itemsPerPage = 50 try { - const totalUsers = await userCount() - const maxPages = Math.ceil(totalUsers / itemsPerPage) + // const totalUsers = await userCount() + // const maxPages = Math.ceil(totalUsers / itemsPerPage) - const users = await userList(pageNumber, itemsPerPage) + const users = await userList({ nextKey, itemsPerPage }) + const newNextKey = users[users.length - 1]?._id res.render('users', { users, - maxPages: Array.from({ length: maxPages }, (_, index) => index + 1), + nextKey: newNextKey, + // maxPages: Array.from({ length: maxPages }, (_, index) => index + 1), }) } catch (error) { + console.error(error) res.status(500).send('Error fetching user data') } } diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index eb501e1..e3001a6 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -1,4 +1,5 @@ import { Session, User } from '@linux-odyssey/models' +import mongoose from 'mongoose' function loginMethods(user) { const methods = [] @@ -11,10 +12,14 @@ export function userCount() { return User.find().count() } -export async function userList(pageNumber, itemsPerPage) { - const skipAmount = (pageNumber - 1) * itemsPerPage - +export async function userList({ nextKey, itemsPerPage }) { + const key = new mongoose.Types.ObjectId(nextKey) + // const matchStage = nextKey ? { user: { $gt: key } } : {} + const matchStage = nextKey ? { user: { $gt: key } } : {} const users = await Session.aggregate([ + { + $match: matchStage, + }, { $group: { _id: '$user', @@ -31,14 +36,14 @@ export async function userList(pageNumber, itemsPerPage) { }, }, { $unwind: '$userData' }, - { $sort: { 'userData.createdAt': 1 } }, - { $skip: skipAmount }, + { $sort: { 'userData._id': 1 } }, { $limit: itemsPerPage }, ]) return users.map(({ userData, count, lastActivityAt }) => { - const { username, email, createdAt } = userData + const { _id, username, email, createdAt } = userData return { + _id, username, email, createdAt: createdAt?.toLocaleString(), diff --git a/packages/analytics/views/users.handlebars b/packages/analytics/views/users.handlebars index fcbedf7..8cb407f 100644 --- a/packages/analytics/views/users.handlebars +++ b/packages/analytics/views/users.handlebars @@ -22,7 +22,5 @@
\ No newline at end of file From c6733814ba0216dfe8d3eaac3d258eed8f90e8de Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Wed, 13 Dec 2023 15:05:25 +0800 Subject: [PATCH 09/19] feat: add sorting order --- packages/analytics/src/constants.js | 23 +++++++++++++++++++++ packages/analytics/src/controllers/users.js | 7 ++++++- packages/analytics/src/models/users.js | 16 ++++++++------ 3 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 packages/analytics/src/constants.js diff --git a/packages/analytics/src/constants.js b/packages/analytics/src/constants.js new file mode 100644 index 0000000..15f98c8 --- /dev/null +++ b/packages/analytics/src/constants.js @@ -0,0 +1,23 @@ +/* eslint-disable class-methods-use-this */ +// eslint-disable-next-line max-classes-per-file +class Order {} + +export class Asc extends Order { + matchStage(nextKey) { + return { $gt: nextKey } + } + + order() { + return 1 + } +} + +export class Desc extends Order { + matchStage(nextKey) { + return { $lt: nextKey } + } + + order() { + return -1 + } +} diff --git a/packages/analytics/src/controllers/users.js b/packages/analytics/src/controllers/users.js index 55894d1..e564a52 100644 --- a/packages/analytics/src/controllers/users.js +++ b/packages/analytics/src/controllers/users.js @@ -1,3 +1,4 @@ +import { Desc } from '../constants.js' import { userList, userCount } from '../models/users.js' export async function userListController(req, res) { @@ -8,7 +9,11 @@ export async function userListController(req, res) { // const totalUsers = await userCount() // const maxPages = Math.ceil(totalUsers / itemsPerPage) - const users = await userList({ nextKey, itemsPerPage }) + const users = await userList({ + nextKey, + itemsPerPage, + order: new Desc(), + }) const newNextKey = users[users.length - 1]?._id res.render('users', { diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index e3001a6..adca652 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -1,5 +1,5 @@ -import { Session, User } from '@linux-odyssey/models' import mongoose from 'mongoose' +import { Session, User } from '@linux-odyssey/models' function loginMethods(user) { const methods = [] @@ -12,10 +12,14 @@ export function userCount() { return User.find().count() } -export async function userList({ nextKey, itemsPerPage }) { - const key = new mongoose.Types.ObjectId(nextKey) - // const matchStage = nextKey ? { user: { $gt: key } } : {} - const matchStage = nextKey ? { user: { $gt: key } } : {} +export async function userList({ nextKey, itemsPerPage, order }) { + console.log(order) + let matchStage = {} + if (nextKey) { + const key = new mongoose.Types.ObjectId(nextKey) + matchStage = { user: order.matchStage(key) } + } + console.log(matchStage) const users = await Session.aggregate([ { $match: matchStage, @@ -36,7 +40,7 @@ export async function userList({ nextKey, itemsPerPage }) { }, }, { $unwind: '$userData' }, - { $sort: { 'userData._id': 1 } }, + { $sort: { 'userData._id': order.order() } }, { $limit: itemsPerPage }, ]) From ceb9416fab222ff4a93a59b12854bc09bc4d7675 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Wed, 13 Dec 2023 15:05:46 +0800 Subject: [PATCH 10/19] update: mongoose version --- packages/analytics/package.json | 3 +- packages/models/package.json | 2 +- server/package.json | 2 +- yarn.lock | 88 ++++++--------------------------- 4 files changed, 20 insertions(+), 75 deletions(-) diff --git a/packages/analytics/package.json b/packages/analytics/package.json index d035121..04928f5 100644 --- a/packages/analytics/package.json +++ b/packages/analytics/package.json @@ -13,7 +13,8 @@ "@linux-odyssey/models": "*", "express": "^4.18.2", "express-basic-auth": "^1.2.1", - "express-handlebars": "^7.1.2" + "express-handlebars": "^7.1.2", + "mongoose": "^8.0.3" }, "devDependencies": { "nodemon": "^3.0.1" diff --git a/packages/models/package.json b/packages/models/package.json index d5f58b0..74077dd 100644 --- a/packages/models/package.json +++ b/packages/models/package.json @@ -6,6 +6,6 @@ "license": "GPL", "exports": "./index.js", "dependencies": { - "mongoose": "^7.6.0" + "mongoose": "^8.0.3" } } diff --git a/server/package.json b/server/package.json index b127f77..9159c9b 100644 --- a/server/package.json +++ b/server/package.json @@ -26,7 +26,7 @@ "express-validator": "^7.0.1", "jsonwebtoken": "^9.0.1", "minimist": "^1.2.8", - "mongoose": "^7.4.0", + "mongoose": "^8.0.3", "node-schedule": "^2.1.1", "passport": "^0.6.0", "passport-github2": "^0.1.12", diff --git a/yarn.lock b/yarn.lock index fb9b082..4b634f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1756,15 +1756,10 @@ bser@2.1.1: dependencies: node-int64 "^0.4.0" -bson@^5.4.0: - version "5.4.0" - resolved "https://registry.yarnpkg.com/bson/-/bson-5.4.0.tgz#0eea77276d490953ad8616b483298dbff07384c6" - integrity sha512-WRZ5SQI5GfUuKnPTNmAYPiKIof3ORXAF4IRU5UcgmivNIon01rWQlw5RUH954dpu8yGL8T59YShVddIPaU/gFA== - -bson@^5.5.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/bson/-/bson-5.5.0.tgz#a419cc69f368d2def3b8b22ea03ed1c9be40e53f" - integrity sha512-B+QB4YmDx9RStKv8LLSl/aVIEV3nYJc3cJNNTK2Cd1TL+7P+cNpw9mAPeCgc5K+j01Dv6sxUzcITXDx7ZU3F0w== +bson@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-6.2.0.tgz#4b6acafc266ba18eeee111373c2699304a9ba0a3" + integrity sha512-ID1cI+7bazPDyL9wYy9GaQ8gEEohWvcUl/Yf0dIdutJxnmInEEyCsb4awy/OiBfall7zBA179Pahi3vCdFze3Q== buffer-crc32@~0.2.3: version "0.2.13" @@ -3668,11 +3663,6 @@ internal-slot@^1.0.5: has "^1.0.3" side-channel "^1.0.4" -ip@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" - integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== - ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -4833,49 +4823,23 @@ mongodb-connection-string-url@^2.6.0: "@types/whatwg-url" "^8.2.1" whatwg-url "^11.0.0" -mongodb@5.7.0: - version "5.7.0" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-5.7.0.tgz#e16d2fcdfd9f8503ec2d88288392dc3235bb3ecc" - integrity sha512-zm82Bq33QbqtxDf58fLWBwTjARK3NSvKYjyz997KSy6hpat0prjeX/kxjbPVyZY60XYPDNETaHkHJI2UCzSLuw== +mongodb@6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-6.2.0.tgz#2c9dcb3eeaf528ed850e94b3df392de6c6b0d7ab" + integrity sha512-d7OSuGjGWDZ5usZPqfvb36laQ9CPhnWkAGHT61x5P95p/8nMVeH8asloMwW6GcYFeB0Vj4CB/1wOTDG2RA9BFA== dependencies: - bson "^5.4.0" - mongodb-connection-string-url "^2.6.0" - socks "^2.7.1" - optionalDependencies: - saslprep "^1.0.3" - -mongodb@5.9.0: - version "5.9.0" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-5.9.0.tgz#5a22065fa8cfaf1d58bf2e3c451cd2c4bfa983a2" - integrity sha512-g+GCMHN1CoRUA+wb1Agv0TI4YTSiWr42B5ulkiAfLLHitGK1R+PkSAf3Lr5rPZwi/3F04LiaZEW0Kxro9Fi2TA== - dependencies: - bson "^5.5.0" - mongodb-connection-string-url "^2.6.0" - socks "^2.7.1" - optionalDependencies: "@mongodb-js/saslprep" "^1.1.0" + bson "^6.2.0" + mongodb-connection-string-url "^2.6.0" -mongoose@^7.4.0: - version "7.4.0" - resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-7.4.0.tgz#3726a8dea0c9e2d54651845cc84c813a6507906a" - integrity sha512-oHE1eqodfKzugXRlQxpo+msIea7jPcRoayDuEMr50+bYwM/juA5f+1stjkWlXcg6vo1PdJFVA6DGaKOPLuG5mA== +mongoose@^8.0.3: + version "8.0.3" + resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-8.0.3.tgz#f3708ba0f40b6dec8f5f1a5d40abd69a80207114" + integrity sha512-LJRT0yP4TW14HT4r2RkxqyvoTylMSzWpl5QOeVHTnRggCLQSpkoBdgbUtORFq/mSL2o9cLCPJz+6uzFj25qbHw== dependencies: - bson "^5.4.0" + bson "^6.2.0" kareem "2.5.1" - mongodb "5.7.0" - mpath "0.9.0" - mquery "5.0.0" - ms "2.1.3" - sift "16.0.1" - -mongoose@^7.6.0: - version "7.6.0" - resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-7.6.0.tgz#3037b754ccdf168385e52d4bcabeb7f5c3b17a5f" - integrity sha512-ztQ12rm0BQN5i7LB6xhWX4l9a9w2aa3jEwa/mM2vAutYJRyAwOzcusvKJBULMzFHyUDBOVW15grisexypgMIWA== - dependencies: - bson "^5.4.0" - kareem "2.5.1" - mongodb "5.9.0" + mongodb "6.2.0" mpath "0.9.0" mquery "5.0.0" ms "2.1.3" @@ -5758,13 +5722,6 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -saslprep@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226" - integrity sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag== - dependencies: - sparse-bitfield "^3.0.3" - semver@^6.0.0, semver@^6.3.0, semver@^6.3.1: version "6.3.1" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" @@ -5913,11 +5870,6 @@ slice-ansi@^4.0.0: astral-regex "^2.0.0" is-fullwidth-code-point "^3.0.0" -smart-buffer@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" - integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== - socket.io-adapter@~2.5.2: version "2.5.2" resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz#5de9477c9182fdc171cd8c8364b9a8894ec75d12" @@ -5956,14 +5908,6 @@ socket.io@^4.7.1: socket.io-adapter "~2.5.2" socket.io-parser "~4.2.4" -socks@^2.7.1: - version "2.7.1" - resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.1.tgz#d8e651247178fde79c0663043e07240196857d55" - integrity sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ== - dependencies: - ip "^2.0.0" - smart-buffer "^4.2.0" - sorted-array-functions@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz#8605695563294dffb2c9796d602bd8459f7a0dd5" From 8c634c8115be55fcf88e2894af37b4c0a82c727e Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Wed, 20 Dec 2023 15:05:10 +0800 Subject: [PATCH 11/19] wip: models/commands analytics --- .../analytics/src/controllers/commands.js | 29 ++++++++++--------- .../analytics/src/controllers/sessions.js | 13 ++++----- packages/analytics/src/models/commands.js | 28 ++++++++++++------ packages/analytics/src/models/sessions.js | 14 ++++++--- packages/analytics/views/commands.handlebars | 4 +-- packages/analytics/views/sessions.handlebars | 4 +-- .../analytics/views/userDetail.handlebars | 0 7 files changed, 52 insertions(+), 40 deletions(-) delete mode 100644 packages/analytics/views/userDetail.handlebars diff --git a/packages/analytics/src/controllers/commands.js b/packages/analytics/src/controllers/commands.js index 514a891..c270ed4 100644 --- a/packages/analytics/src/controllers/commands.js +++ b/packages/analytics/src/controllers/commands.js @@ -4,25 +4,26 @@ import { errorCommands } from '../models/commands.js' // eslint-disable-next-line import/prefer-default-export export async function commandListController(req, res) { + const { nextKey } = req.query const itemsPerPage = 250 try { - const totalCommands = await Command.countDocuments({ - $and: [ - { error: { $exists: true } }, - { error: { $ne: null } }, - { error: { $ne: '' } }, - ], - }) + // const totalCommands = await Command.countDocuments({ + // $and: [ + // { error: { $exists: true } }, + // { error: { $ne: null } }, + // { error: { $ne: '' } }, + // ], + // }) - const maxPages = Math.ceil(totalCommands / itemsPerPage) + // const maxPages = Math.ceil(totalCommands / itemsPerPage) - const pageNumber = req.query.page || 1 - const payload = { - errorCommands: await errorCommands(pageNumber, itemsPerPage), - pages: Array.from({ length: maxPages }, (_, index) => index + 1), - } - res.render('commands', payload) + // const pageNumber = req.query + + const payload = await errorCommands({ nextKey, itemsPerPage }) + const newNextKey = payload[payload.length - 1]?._id + + res.render('commands', { payload, nextKey: newNextKey }) } catch (error) { res.status(500).send('Error fetching data') } diff --git a/packages/analytics/src/controllers/sessions.js b/packages/analytics/src/controllers/sessions.js index e0b131b..238aee0 100644 --- a/packages/analytics/src/controllers/sessions.js +++ b/packages/analytics/src/controllers/sessions.js @@ -1,18 +1,17 @@ import { sessionCount, sessionList, sessionDetail } from '../models/sessions.js' export async function sessionListController(req, res) { - const pageNumber = req.query.page || 1 + const { nextKey } = req.query const itemsPerPage = 100 try { - const totalSessions = await sessionCount() - const maxPages = Math.ceil(totalSessions / itemsPerPage) + // const totalSessions = await sessionCount() + // const maxPages = Math.ceil(totalSessions / itemsPerPage) - const sessions = await sessionList(pageNumber, itemsPerPage) + const sessions = await sessionList({ nextKey, itemsPerPage }) + const newNextKey = sessions[sessions.length - 1]?._id - const pages = Array.from({ length: maxPages }, (_, index) => index + 1) - - res.render('sessions', { sessions, pages }) + res.render('sessions', { sessions, nextKey: newNextKey }) } catch (error) { res.status(500).send('Error fetching session data') } diff --git a/packages/analytics/src/models/commands.js b/packages/analytics/src/models/commands.js index c2a756a..a6a3b0a 100644 --- a/packages/analytics/src/models/commands.js +++ b/packages/analytics/src/models/commands.js @@ -1,16 +1,26 @@ import { Command } from '@linux-odyssey/models' +import mongoose from 'mongoose' // eslint-disable-next-line import/prefer-default-export -export async function errorCommands(pageNumber, itemsPerPage) { - const skipAmount = (pageNumber - 1) * itemsPerPage +export async function errorCommands({ nextKey, itemsPerPage }) { + const key = new mongoose.Types.ObjectId(nextKey) + const matchStage = nextKey ? { _id: { $lt: key } } : {} - const commands = await Command.find({ - $and: [ - { error: { $exists: true } }, - { error: { $ne: null } }, - { error: { $ne: '' } }, - ], - }) + const commands = await Command.aggregate([ + { + $match: matchStage, + }, + { + $match: { + $and: [ + { error: { $exists: true } }, + { error: { $ne: null } }, + { error: { $ne: '' } }, + ], + } + } + { $limit: itemsPerPage }, + ]) .populate({ path: 'session', populate: { path: 'user' }, diff --git a/packages/analytics/src/models/sessions.js b/packages/analytics/src/models/sessions.js index a5568d8..e357f96 100644 --- a/packages/analytics/src/models/sessions.js +++ b/packages/analytics/src/models/sessions.js @@ -1,4 +1,5 @@ import { Command, Session } from '@linux-odyssey/models' +import mongoose from 'mongoose' export function sessionCount() { return Session.find().count() @@ -30,10 +31,14 @@ function formatTime(time) { return `${hours}h ${minutes % 60}m ${seconds % 60}s` } -export async function sessionList(pageNumber, itemsPerPage) { - const skipAmount = (pageNumber - 1) * itemsPerPage +export async function sessionList({ nextKey, itemsPerPage }) { + const key = new mongoose.Types.ObjectId(nextKey) + const matchStage = nextKey ? { _id: { $lt: key } } : {} const sessions = await Session.aggregate([ + { + $match: matchStage, + }, { $lookup: { from: 'users', @@ -51,9 +56,10 @@ export async function sessionList(pageNumber, itemsPerPage) { as: 'commands', }, }, + { $sort: { _id: -1 } }, + { $limit: itemsPerPage }, ]) - .skip(skipAmount) - .limit(itemsPerPage) + return sessions.map( ({ _id, diff --git a/packages/analytics/views/commands.handlebars b/packages/analytics/views/commands.handlebars index 00843c9..1d7034f 100644 --- a/packages/analytics/views/commands.handlebars +++ b/packages/analytics/views/commands.handlebars @@ -23,7 +23,5 @@ \ No newline at end of file diff --git a/packages/analytics/views/sessions.handlebars b/packages/analytics/views/sessions.handlebars index a9abf8c..236795f 100644 --- a/packages/analytics/views/sessions.handlebars +++ b/packages/analytics/views/sessions.handlebars @@ -26,7 +26,5 @@ \ No newline at end of file diff --git a/packages/analytics/views/userDetail.handlebars b/packages/analytics/views/userDetail.handlebars deleted file mode 100644 index e69de29..0000000 From bc0055781ef1f57365efb2c89a9c702436621a7e Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 16:25:44 +0800 Subject: [PATCH 12/19] feat: finish errorCommands pagination --- .../analytics/src/controllers/commands.js | 11 ++-- .../analytics/src/controllers/sessions.js | 10 +++- packages/analytics/src/controllers/users.js | 9 +--- packages/analytics/src/models/commands.js | 51 ++++++++++++------- packages/analytics/src/models/sessions.js | 14 ++--- packages/analytics/src/models/users.js | 5 +- 6 files changed, 59 insertions(+), 41 deletions(-) diff --git a/packages/analytics/src/controllers/commands.js b/packages/analytics/src/controllers/commands.js index c270ed4..4ba78a2 100644 --- a/packages/analytics/src/controllers/commands.js +++ b/packages/analytics/src/controllers/commands.js @@ -1,11 +1,12 @@ /* eslint-disable object-shorthand */ +import { Desc } from '../constants.js' import { Command } from '@linux-odyssey/models' import { errorCommands } from '../models/commands.js' // eslint-disable-next-line import/prefer-default-export export async function commandListController(req, res) { const { nextKey } = req.query - const itemsPerPage = 250 + const itemsPerPage = 100 try { // const totalCommands = await Command.countDocuments({ @@ -20,10 +21,14 @@ export async function commandListController(req, res) { // const pageNumber = req.query - const payload = await errorCommands({ nextKey, itemsPerPage }) + const payload = await errorCommands({ + nextKey, + itemsPerPage, + order: new Desc(), + }) const newNextKey = payload[payload.length - 1]?._id - res.render('commands', { payload, nextKey: newNextKey }) + res.render('commands', { errorCommands: payload, nextKey: newNextKey }) } catch (error) { res.status(500).send('Error fetching data') } diff --git a/packages/analytics/src/controllers/sessions.js b/packages/analytics/src/controllers/sessions.js index 238aee0..3c603bb 100644 --- a/packages/analytics/src/controllers/sessions.js +++ b/packages/analytics/src/controllers/sessions.js @@ -1,18 +1,24 @@ +import { Desc } from '../constants.js' import { sessionCount, sessionList, sessionDetail } from '../models/sessions.js' export async function sessionListController(req, res) { const { nextKey } = req.query - const itemsPerPage = 100 + const itemsPerPage = 10 try { // const totalSessions = await sessionCount() // const maxPages = Math.ceil(totalSessions / itemsPerPage) - const sessions = await sessionList({ nextKey, itemsPerPage }) + const sessions = await sessionList({ + nextKey, + itemsPerPage, + order: new Desc(), + }) const newNextKey = sessions[sessions.length - 1]?._id res.render('sessions', { sessions, nextKey: newNextKey }) } catch (error) { + console.log(error) res.status(500).send('Error fetching session data') } } diff --git a/packages/analytics/src/controllers/users.js b/packages/analytics/src/controllers/users.js index e564a52..0e81a83 100644 --- a/packages/analytics/src/controllers/users.js +++ b/packages/analytics/src/controllers/users.js @@ -6,9 +6,6 @@ export async function userListController(req, res) { const itemsPerPage = 50 try { - // const totalUsers = await userCount() - // const maxPages = Math.ceil(totalUsers / itemsPerPage) - const users = await userList({ nextKey, itemsPerPage, @@ -16,11 +13,7 @@ export async function userListController(req, res) { }) const newNextKey = users[users.length - 1]?._id - res.render('users', { - users, - nextKey: newNextKey, - // maxPages: Array.from({ length: maxPages }, (_, index) => index + 1), - }) + res.render('users', { users, nextKey: newNextKey }) } catch (error) { console.error(error) res.status(500).send('Error fetching user data') diff --git a/packages/analytics/src/models/commands.js b/packages/analytics/src/models/commands.js index a6a3b0a..89d1b2d 100644 --- a/packages/analytics/src/models/commands.js +++ b/packages/analytics/src/models/commands.js @@ -1,15 +1,16 @@ -import { Command } from '@linux-odyssey/models' import mongoose from 'mongoose' +import { Command } from '@linux-odyssey/models' // eslint-disable-next-line import/prefer-default-export -export async function errorCommands({ nextKey, itemsPerPage }) { - const key = new mongoose.Types.ObjectId(nextKey) - const matchStage = nextKey ? { _id: { $lt: key } } : {} - +export async function errorCommands({ nextKey, itemsPerPage, order }) { + let matchStage = {} + if (nextKey) { + const key = new mongoose.Types.ObjectId(nextKey) + matchStage = { _id: order.matchStage(key) } + } + console.log(matchStage, itemsPerPage, nextKey, order) const commands = await Command.aggregate([ - { - $match: matchStage, - }, + { $match: matchStage }, { $match: { $and: [ @@ -17,20 +18,34 @@ export async function errorCommands({ nextKey, itemsPerPage }) { { error: { $ne: null } }, { error: { $ne: '' } }, ], - } - } + }, + }, + { + $lookup: { + from: 'sessions', // 替換成 'session' 對應的集合名稱 + localField: 'session', // 替換成 'session' 對應的字段名稱 + foreignField: '_id', + as: 'session', + }, + }, + { $unwind: '$session' }, + { + $lookup: { + from: 'users', // 替換成 'user' 對應的集合名稱 + localField: 'session.user', // 替換成 'user' 對應的字段名稱 + foreignField: '_id', + as: 'session.user', + }, + }, + { $unwind: '$session.user' }, + { $sort: { _id: order.order() } }, { $limit: itemsPerPage }, ]) - .populate({ - path: 'session', - populate: { path: 'user' }, - }) - .skip(skipAmount) - .limit(itemsPerPage) - .exec() - return commands.map(({ command, error, createdAt, session }) => { + + return commands.map(({ _id, command, error, createdAt, session }) => { const { user, quest } = session return { + _id, command: command.slice(0, 20), error: error.slice(0, 50), createdAt: createdAt?.toLocaleString(), diff --git a/packages/analytics/src/models/sessions.js b/packages/analytics/src/models/sessions.js index e357f96..eb69785 100644 --- a/packages/analytics/src/models/sessions.js +++ b/packages/analytics/src/models/sessions.js @@ -1,5 +1,5 @@ -import { Command, Session } from '@linux-odyssey/models' import mongoose from 'mongoose' +import { Command, Session } from '@linux-odyssey/models' export function sessionCount() { return Session.find().count() @@ -31,10 +31,12 @@ function formatTime(time) { return `${hours}h ${minutes % 60}m ${seconds % 60}s` } -export async function sessionList({ nextKey, itemsPerPage }) { - const key = new mongoose.Types.ObjectId(nextKey) - const matchStage = nextKey ? { _id: { $lt: key } } : {} - +export async function sessionList({ nextKey, itemsPerPage, order }) { + let matchStage = {} + if (nextKey) { + const key = new mongoose.Types.ObjectId(nextKey) + matchStage = { user: order.matchStage(key) } + } const sessions = await Session.aggregate([ { $match: matchStage, @@ -56,7 +58,7 @@ export async function sessionList({ nextKey, itemsPerPage }) { as: 'commands', }, }, - { $sort: { _id: -1 } }, + { $sort: { _id: order.order() } }, { $limit: itemsPerPage }, ]) diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index adca652..34075b3 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -13,7 +13,6 @@ export function userCount() { } export async function userList({ nextKey, itemsPerPage, order }) { - console.log(order) let matchStage = {} if (nextKey) { const key = new mongoose.Types.ObjectId(nextKey) @@ -21,9 +20,7 @@ export async function userList({ nextKey, itemsPerPage, order }) { } console.log(matchStage) const users = await Session.aggregate([ - { - $match: matchStage, - }, + { $match: matchStage }, { $group: { _id: '$user', From 19cd10ab7bea1f8e81e3af2d493f467bb7ee4efb Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 16:37:37 +0800 Subject: [PATCH 13/19] fix: session pagination --- packages/analytics/src/controllers/sessions.js | 2 +- packages/analytics/src/models/sessions.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/analytics/src/controllers/sessions.js b/packages/analytics/src/controllers/sessions.js index 3c603bb..8cbe537 100644 --- a/packages/analytics/src/controllers/sessions.js +++ b/packages/analytics/src/controllers/sessions.js @@ -3,7 +3,7 @@ import { sessionCount, sessionList, sessionDetail } from '../models/sessions.js' export async function sessionListController(req, res) { const { nextKey } = req.query - const itemsPerPage = 10 + const itemsPerPage = 50 try { // const totalSessions = await sessionCount() diff --git a/packages/analytics/src/models/sessions.js b/packages/analytics/src/models/sessions.js index eb69785..d2a47c6 100644 --- a/packages/analytics/src/models/sessions.js +++ b/packages/analytics/src/models/sessions.js @@ -35,7 +35,7 @@ export async function sessionList({ nextKey, itemsPerPage, order }) { let matchStage = {} if (nextKey) { const key = new mongoose.Types.ObjectId(nextKey) - matchStage = { user: order.matchStage(key) } + matchStage = { _id: order.matchStage(key) } } const sessions = await Session.aggregate([ { From 68db16419dc492bfc9f13bd9ff280251d08c6b8c Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 17:09:20 +0800 Subject: [PATCH 14/19] refactor: add Pagination class --- packages/analytics/src/constants.js | 23 ---------- .../analytics/src/controllers/commands.js | 21 +--------- .../analytics/src/controllers/sessions.js | 10 ++--- packages/analytics/src/controllers/users.js | 10 ++--- packages/analytics/src/models/commands.js | 15 ++----- packages/analytics/src/models/pagination.js | 42 +++++++++++++++++++ packages/analytics/src/models/sessions.js | 16 ++----- packages/analytics/src/models/users.js | 14 ++----- packages/analytics/tests/pagination.test.js | 37 ++++++++++++++++ 9 files changed, 99 insertions(+), 89 deletions(-) create mode 100644 packages/analytics/src/models/pagination.js create mode 100644 packages/analytics/tests/pagination.test.js diff --git a/packages/analytics/src/constants.js b/packages/analytics/src/constants.js index 15f98c8..e69de29 100644 --- a/packages/analytics/src/constants.js +++ b/packages/analytics/src/constants.js @@ -1,23 +0,0 @@ -/* eslint-disable class-methods-use-this */ -// eslint-disable-next-line max-classes-per-file -class Order {} - -export class Asc extends Order { - matchStage(nextKey) { - return { $gt: nextKey } - } - - order() { - return 1 - } -} - -export class Desc extends Order { - matchStage(nextKey) { - return { $lt: nextKey } - } - - order() { - return -1 - } -} diff --git a/packages/analytics/src/controllers/commands.js b/packages/analytics/src/controllers/commands.js index 4ba78a2..3a92b0a 100644 --- a/packages/analytics/src/controllers/commands.js +++ b/packages/analytics/src/controllers/commands.js @@ -1,6 +1,5 @@ /* eslint-disable object-shorthand */ -import { Desc } from '../constants.js' -import { Command } from '@linux-odyssey/models' +import { Desc } from '../models/pagination.js' import { errorCommands } from '../models/commands.js' // eslint-disable-next-line import/prefer-default-export @@ -9,23 +8,7 @@ export async function commandListController(req, res) { const itemsPerPage = 100 try { - // const totalCommands = await Command.countDocuments({ - // $and: [ - // { error: { $exists: true } }, - // { error: { $ne: null } }, - // { error: { $ne: '' } }, - // ], - // }) - - // const maxPages = Math.ceil(totalCommands / itemsPerPage) - - // const pageNumber = req.query - - const payload = await errorCommands({ - nextKey, - itemsPerPage, - order: new Desc(), - }) + const payload = await errorCommands(new Desc(itemsPerPage, nextKey)) const newNextKey = payload[payload.length - 1]?._id res.render('commands', { errorCommands: payload, nextKey: newNextKey }) diff --git a/packages/analytics/src/controllers/sessions.js b/packages/analytics/src/controllers/sessions.js index 8cbe537..435dc74 100644 --- a/packages/analytics/src/controllers/sessions.js +++ b/packages/analytics/src/controllers/sessions.js @@ -1,5 +1,5 @@ -import { Desc } from '../constants.js' -import { sessionCount, sessionList, sessionDetail } from '../models/sessions.js' +import { Desc } from '../models/pagination.js' +import { sessionList, sessionDetail } from '../models/sessions.js' export async function sessionListController(req, res) { const { nextKey } = req.query @@ -9,11 +9,7 @@ export async function sessionListController(req, res) { // const totalSessions = await sessionCount() // const maxPages = Math.ceil(totalSessions / itemsPerPage) - const sessions = await sessionList({ - nextKey, - itemsPerPage, - order: new Desc(), - }) + const sessions = await sessionList(new Desc(itemsPerPage, nextKey)) const newNextKey = sessions[sessions.length - 1]?._id res.render('sessions', { sessions, nextKey: newNextKey }) diff --git a/packages/analytics/src/controllers/users.js b/packages/analytics/src/controllers/users.js index 0e81a83..69ebf91 100644 --- a/packages/analytics/src/controllers/users.js +++ b/packages/analytics/src/controllers/users.js @@ -1,16 +1,12 @@ -import { Desc } from '../constants.js' -import { userList, userCount } from '../models/users.js' +import { Desc } from '../models/pagination.js' +import { userList } from '../models/users.js' export async function userListController(req, res) { const { nextKey } = req.query const itemsPerPage = 50 try { - const users = await userList({ - nextKey, - itemsPerPage, - order: new Desc(), - }) + const users = await userList(new Desc(itemsPerPage, nextKey)) const newNextKey = users[users.length - 1]?._id res.render('users', { users, nextKey: newNextKey }) diff --git a/packages/analytics/src/models/commands.js b/packages/analytics/src/models/commands.js index 89d1b2d..32bc9ae 100644 --- a/packages/analytics/src/models/commands.js +++ b/packages/analytics/src/models/commands.js @@ -1,16 +1,9 @@ -import mongoose from 'mongoose' import { Command } from '@linux-odyssey/models' // eslint-disable-next-line import/prefer-default-export -export async function errorCommands({ nextKey, itemsPerPage, order }) { - let matchStage = {} - if (nextKey) { - const key = new mongoose.Types.ObjectId(nextKey) - matchStage = { _id: order.matchStage(key) } - } - console.log(matchStage, itemsPerPage, nextKey, order) +export async function errorCommands(pagination) { const commands = await Command.aggregate([ - { $match: matchStage }, + pagination.matchStage('_id'), { $match: { $and: [ @@ -38,8 +31,8 @@ export async function errorCommands({ nextKey, itemsPerPage, order }) { }, }, { $unwind: '$session.user' }, - { $sort: { _id: order.order() } }, - { $limit: itemsPerPage }, + pagination.order('_id'), + pagination.limit(), ]) return commands.map(({ _id, command, error, createdAt, session }) => { diff --git a/packages/analytics/src/models/pagination.js b/packages/analytics/src/models/pagination.js new file mode 100644 index 0000000..4509597 --- /dev/null +++ b/packages/analytics/src/models/pagination.js @@ -0,0 +1,42 @@ +/* eslint-disable lines-between-class-members */ +/* eslint-disable class-methods-use-this */ +// eslint-disable-next-line max-classes-per-file +import mongoose from 'mongoose' + +class Pagination { + nextKey = null + itemsPerPage = 50 + + constructor(itemsPerPage = 50, nextKey = null) { + this.itemsPerPage = itemsPerPage + if (nextKey) { + this.nextKey = new mongoose.Types.ObjectId(nextKey) + } + } + + limit() { + return { $limit: this.itemsPerPage } + } +} + +export class Asc extends Pagination { + matchStage(field) { + const stage = this.nextKey ? { [field]: { $gt: this.nextKey } } : {} + return { $match: stage } + } + + order(field) { + return { $sort: { [field]: 1 } } + } +} + +export class Desc extends Pagination { + matchStage(field) { + const stage = this.nextKey ? { [field]: { $lt: this.nextKey } } : {} + return { $match: stage } + } + + order(field) { + return { $sort: { [field]: -1 } } + } +} diff --git a/packages/analytics/src/models/sessions.js b/packages/analytics/src/models/sessions.js index d2a47c6..09c7357 100644 --- a/packages/analytics/src/models/sessions.js +++ b/packages/analytics/src/models/sessions.js @@ -1,4 +1,3 @@ -import mongoose from 'mongoose' import { Command, Session } from '@linux-odyssey/models' export function sessionCount() { @@ -31,16 +30,9 @@ function formatTime(time) { return `${hours}h ${minutes % 60}m ${seconds % 60}s` } -export async function sessionList({ nextKey, itemsPerPage, order }) { - let matchStage = {} - if (nextKey) { - const key = new mongoose.Types.ObjectId(nextKey) - matchStage = { _id: order.matchStage(key) } - } +export async function sessionList(pagination) { const sessions = await Session.aggregate([ - { - $match: matchStage, - }, + pagination.matchStage('_id'), { $lookup: { from: 'users', @@ -58,8 +50,8 @@ export async function sessionList({ nextKey, itemsPerPage, order }) { as: 'commands', }, }, - { $sort: { _id: order.order() } }, - { $limit: itemsPerPage }, + pagination.order('_id'), + pagination.limit(), ]) return sessions.map( diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index 34075b3..97ac208 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -12,15 +12,9 @@ export function userCount() { return User.find().count() } -export async function userList({ nextKey, itemsPerPage, order }) { - let matchStage = {} - if (nextKey) { - const key = new mongoose.Types.ObjectId(nextKey) - matchStage = { user: order.matchStage(key) } - } - console.log(matchStage) +export async function userList(pagination) { const users = await Session.aggregate([ - { $match: matchStage }, + pagination.matchStage('user'), { $group: { _id: '$user', @@ -37,8 +31,8 @@ export async function userList({ nextKey, itemsPerPage, order }) { }, }, { $unwind: '$userData' }, - { $sort: { 'userData._id': order.order() } }, - { $limit: itemsPerPage }, + pagination.order('userData._id'), + pagination.limit(), ]) return users.map(({ userData, count, lastActivityAt }) => { diff --git a/packages/analytics/tests/pagination.test.js b/packages/analytics/tests/pagination.test.js new file mode 100644 index 0000000..c43dccd --- /dev/null +++ b/packages/analytics/tests/pagination.test.js @@ -0,0 +1,37 @@ +import mongoose from 'mongoose' +import { Desc } from '../src/models/pagination.js' + +describe('Pagination', () => { + it('should return correct limit', async () => { + const pagination = new Desc(10) + + expect(pagination.limit()).toMatchObject({ $limit: 10 }) + }) + + it('should return correct order', async () => { + const pagination = new Desc(10) + + expect(pagination.order('_id')).toMatchObject({ $sort: { _id: -1 } }) + expect(pagination.order('time')).toMatchObject({ $sort: { time: -1 } }) + }) + + it('should return correct match stage for empty nextKey', async () => { + const pagination = new Desc(10) + + expect(pagination.matchStage('_id')).toMatchObject({ $match: {} }) + }) + + it('should return correct match stage for empty nextKey', async () => { + const nextKeyStr = '656c7fea28a4c087bfcefe12' + const pagination = new Desc(10, nextKeyStr) + const nextKey = new mongoose.Types.ObjectId(nextKeyStr) + + expect(pagination.matchStage('_id')).toMatchObject({ + $match: { + _id: { + $lt: nextKey, + }, + }, + }) + }) +}) From fa36b63fbe339fa05f23d80d88635c3750aa8780 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 17:28:48 +0800 Subject: [PATCH 15/19] feat: add order to pagination --- .../analytics/src/controllers/commands.js | 9 ++- .../analytics/src/controllers/sessions.js | 8 ++- packages/analytics/src/controllers/users.js | 6 +- packages/analytics/src/models/commands.js | 4 +- packages/analytics/src/models/pagination.js | 68 ++++++++++++------- packages/analytics/src/models/sessions.js | 4 +- packages/analytics/src/models/users.js | 4 +- packages/analytics/tests/pagination.test.js | 32 ++++++--- 8 files changed, 85 insertions(+), 50 deletions(-) diff --git a/packages/analytics/src/controllers/commands.js b/packages/analytics/src/controllers/commands.js index 3a92b0a..36d2f21 100644 --- a/packages/analytics/src/controllers/commands.js +++ b/packages/analytics/src/controllers/commands.js @@ -1,18 +1,21 @@ /* eslint-disable object-shorthand */ -import { Desc } from '../models/pagination.js' +import Pagination from '../models/pagination.js' import { errorCommands } from '../models/commands.js' // eslint-disable-next-line import/prefer-default-export export async function commandListController(req, res) { - const { nextKey } = req.query + const { nextKey, order } = req.query const itemsPerPage = 100 try { - const payload = await errorCommands(new Desc(itemsPerPage, nextKey)) + const payload = await errorCommands( + new Pagination(order, itemsPerPage, nextKey) + ) const newNextKey = payload[payload.length - 1]?._id res.render('commands', { errorCommands: payload, nextKey: newNextKey }) } catch (error) { + console.error(error) res.status(500).send('Error fetching data') } } diff --git a/packages/analytics/src/controllers/sessions.js b/packages/analytics/src/controllers/sessions.js index 435dc74..765f1f7 100644 --- a/packages/analytics/src/controllers/sessions.js +++ b/packages/analytics/src/controllers/sessions.js @@ -1,15 +1,17 @@ -import { Desc } from '../models/pagination.js' +import Pagination from '../models/pagination.js' import { sessionList, sessionDetail } from '../models/sessions.js' export async function sessionListController(req, res) { - const { nextKey } = req.query + const { nextKey, order } = req.query const itemsPerPage = 50 try { // const totalSessions = await sessionCount() // const maxPages = Math.ceil(totalSessions / itemsPerPage) - const sessions = await sessionList(new Desc(itemsPerPage, nextKey)) + const sessions = await sessionList( + new Pagination(order, itemsPerPage, nextKey) + ) const newNextKey = sessions[sessions.length - 1]?._id res.render('sessions', { sessions, nextKey: newNextKey }) diff --git a/packages/analytics/src/controllers/users.js b/packages/analytics/src/controllers/users.js index 69ebf91..96d198c 100644 --- a/packages/analytics/src/controllers/users.js +++ b/packages/analytics/src/controllers/users.js @@ -1,12 +1,12 @@ -import { Desc } from '../models/pagination.js' +import Pagination from '../models/pagination.js' import { userList } from '../models/users.js' export async function userListController(req, res) { - const { nextKey } = req.query + const { nextKey, order } = req.query const itemsPerPage = 50 try { - const users = await userList(new Desc(itemsPerPage, nextKey)) + const users = await userList(new Pagination(order, itemsPerPage, nextKey)) const newNextKey = users[users.length - 1]?._id res.render('users', { users, nextKey: newNextKey }) diff --git a/packages/analytics/src/models/commands.js b/packages/analytics/src/models/commands.js index 32bc9ae..e771c4f 100644 --- a/packages/analytics/src/models/commands.js +++ b/packages/analytics/src/models/commands.js @@ -3,7 +3,7 @@ import { Command } from '@linux-odyssey/models' // eslint-disable-next-line import/prefer-default-export export async function errorCommands(pagination) { const commands = await Command.aggregate([ - pagination.matchStage('_id'), + pagination.match('_id'), { $match: { $and: [ @@ -31,7 +31,7 @@ export async function errorCommands(pagination) { }, }, { $unwind: '$session.user' }, - pagination.order('_id'), + pagination.sort('_id'), pagination.limit(), ]) diff --git a/packages/analytics/src/models/pagination.js b/packages/analytics/src/models/pagination.js index 4509597..d253bec 100644 --- a/packages/analytics/src/models/pagination.js +++ b/packages/analytics/src/models/pagination.js @@ -3,40 +3,60 @@ // eslint-disable-next-line max-classes-per-file import mongoose from 'mongoose' -class Pagination { +export const DESC = 'desc' +export const ASC = 'asc' + +class Asc { + getOperator() { + return '$gt' + } + + getOrder() { + return 1 + } +} + +class Desc { + getOperator() { + return '$lt' + } + + getOrder() { + return -1 + } +} + +export default class Pagination { nextKey = null itemsPerPage = 50 + order = null - constructor(itemsPerPage = 50, nextKey = null) { + constructor(order, itemsPerPage = 50, nextKey = null) { this.itemsPerPage = itemsPerPage + if (!order || order === DESC) { + this.order = new Desc() + } else if (order === ASC) { + this.order = new Asc() + } else { + throw new Error('order must be asc or desc') + } if (nextKey) { this.nextKey = new mongoose.Types.ObjectId(nextKey) } } + match(field) { + const stage = this.nextKey + ? { [field]: { [this.order.getOperator()]: this.nextKey } } + : {} + return { $match: stage } + } + + sort(field) { + return { $sort: { [field]: this.order.getOrder() } } + } + limit() { return { $limit: this.itemsPerPage } } } - -export class Asc extends Pagination { - matchStage(field) { - const stage = this.nextKey ? { [field]: { $gt: this.nextKey } } : {} - return { $match: stage } - } - - order(field) { - return { $sort: { [field]: 1 } } - } -} - -export class Desc extends Pagination { - matchStage(field) { - const stage = this.nextKey ? { [field]: { $lt: this.nextKey } } : {} - return { $match: stage } - } - - order(field) { - return { $sort: { [field]: -1 } } - } -} diff --git a/packages/analytics/src/models/sessions.js b/packages/analytics/src/models/sessions.js index 09c7357..7bc7022 100644 --- a/packages/analytics/src/models/sessions.js +++ b/packages/analytics/src/models/sessions.js @@ -32,7 +32,7 @@ function formatTime(time) { export async function sessionList(pagination) { const sessions = await Session.aggregate([ - pagination.matchStage('_id'), + pagination.match('_id'), { $lookup: { from: 'users', @@ -50,7 +50,7 @@ export async function sessionList(pagination) { as: 'commands', }, }, - pagination.order('_id'), + pagination.sort('_id'), pagination.limit(), ]) diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index 97ac208..2452123 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -14,7 +14,7 @@ export function userCount() { export async function userList(pagination) { const users = await Session.aggregate([ - pagination.matchStage('user'), + pagination.match('user'), { $group: { _id: '$user', @@ -31,7 +31,7 @@ export async function userList(pagination) { }, }, { $unwind: '$userData' }, - pagination.order('userData._id'), + pagination.sort('userData._id'), pagination.limit(), ]) diff --git a/packages/analytics/tests/pagination.test.js b/packages/analytics/tests/pagination.test.js index c43dccd..9c1db98 100644 --- a/packages/analytics/tests/pagination.test.js +++ b/packages/analytics/tests/pagination.test.js @@ -1,32 +1,42 @@ import mongoose from 'mongoose' -import { Desc } from '../src/models/pagination.js' +import Pagination, { ASC, DESC } from '../src/models/pagination.js' describe('Pagination', () => { - it('should return correct limit', async () => { - const pagination = new Desc(10) + it('should failed when not provide order', () => { + expect(() => new Pagination()).toThrow('order must be asc or desc') + }) + it('should return correct limit', () => { + const pagination = new Pagination(DESC, 10) expect(pagination.limit()).toMatchObject({ $limit: 10 }) }) - it('should return correct order', async () => { - const pagination = new Desc(10) + it('desc should return correct order', async () => { + const pagination = new Pagination(DESC, 10) - expect(pagination.order('_id')).toMatchObject({ $sort: { _id: -1 } }) - expect(pagination.order('time')).toMatchObject({ $sort: { time: -1 } }) + expect(pagination.sort('_id')).toMatchObject({ $sort: { _id: -1 } }) + expect(pagination.sort('time')).toMatchObject({ $sort: { time: -1 } }) + }) + + it('asc should return correct order', async () => { + const pagination = new Pagination(ASC, 10) + + expect(pagination.sort('_id')).toMatchObject({ $sort: { _id: 1 } }) + expect(pagination.sort('time')).toMatchObject({ $sort: { time: 1 } }) }) it('should return correct match stage for empty nextKey', async () => { - const pagination = new Desc(10) + const pagination = new Pagination(DESC, 10) - expect(pagination.matchStage('_id')).toMatchObject({ $match: {} }) + expect(pagination.match('_id')).toMatchObject({ $match: {} }) }) it('should return correct match stage for empty nextKey', async () => { const nextKeyStr = '656c7fea28a4c087bfcefe12' - const pagination = new Desc(10, nextKeyStr) + const pagination = new Pagination(DESC, 10, nextKeyStr) const nextKey = new mongoose.Types.ObjectId(nextKeyStr) - expect(pagination.matchStage('_id')).toMatchObject({ + expect(pagination.match('_id')).toMatchObject({ $match: { _id: { $lt: nextKey, From 3b686931778a0f05152f57dbc66b8e69f9314f09 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 17:48:36 +0800 Subject: [PATCH 16/19] feat: add pagination buttons --- packages/analytics/src/controllers/commands.js | 11 +++++++---- packages/analytics/src/controllers/sessions.js | 14 +++++++------- packages/analytics/src/controllers/users.js | 9 +++++++-- packages/analytics/src/models/pagination.js | 12 ++++++++++++ packages/analytics/tests/pagination.test.js | 3 ++- packages/analytics/views/commands.handlebars | 10 +++++++--- packages/analytics/views/sessions.handlebars | 6 +++++- packages/analytics/views/users.handlebars | 6 +++++- 8 files changed, 52 insertions(+), 19 deletions(-) diff --git a/packages/analytics/src/controllers/commands.js b/packages/analytics/src/controllers/commands.js index 36d2f21..b570504 100644 --- a/packages/analytics/src/controllers/commands.js +++ b/packages/analytics/src/controllers/commands.js @@ -8,12 +8,15 @@ export async function commandListController(req, res) { const itemsPerPage = 100 try { - const payload = await errorCommands( - new Pagination(order, itemsPerPage, nextKey) - ) + const pagination = new Pagination(order, itemsPerPage, nextKey) + const payload = await errorCommands(pagination) const newNextKey = payload[payload.length - 1]?._id - res.render('commands', { errorCommands: payload, nextKey: newNextKey }) + res.render('commands', { + errorCommands: payload, + nextKey: newNextKey, + order: pagination.getOrder(), + }) } catch (error) { console.error(error) res.status(500).send('Error fetching data') diff --git a/packages/analytics/src/controllers/sessions.js b/packages/analytics/src/controllers/sessions.js index 765f1f7..a4dbe89 100644 --- a/packages/analytics/src/controllers/sessions.js +++ b/packages/analytics/src/controllers/sessions.js @@ -6,15 +6,15 @@ export async function sessionListController(req, res) { const itemsPerPage = 50 try { - // const totalSessions = await sessionCount() - // const maxPages = Math.ceil(totalSessions / itemsPerPage) - - const sessions = await sessionList( - new Pagination(order, itemsPerPage, nextKey) - ) + const pagination = new Pagination(order, itemsPerPage, nextKey) + const sessions = await sessionList(pagination) const newNextKey = sessions[sessions.length - 1]?._id - res.render('sessions', { sessions, nextKey: newNextKey }) + res.render('sessions', { + sessions, + nextKey: newNextKey, + order: pagination.getOrder(), + }) } catch (error) { console.log(error) res.status(500).send('Error fetching session data') diff --git a/packages/analytics/src/controllers/users.js b/packages/analytics/src/controllers/users.js index 96d198c..220bb3b 100644 --- a/packages/analytics/src/controllers/users.js +++ b/packages/analytics/src/controllers/users.js @@ -6,10 +6,15 @@ export async function userListController(req, res) { const itemsPerPage = 50 try { - const users = await userList(new Pagination(order, itemsPerPage, nextKey)) + const pagination = new Pagination(order, itemsPerPage, nextKey) + const users = await userList(pagination) const newNextKey = users[users.length - 1]?._id - res.render('users', { users, nextKey: newNextKey }) + res.render('users', { + users, + nextKey: newNextKey, + order: pagination.getOrder(), + }) } catch (error) { console.error(error) res.status(500).send('Error fetching user data') diff --git a/packages/analytics/src/models/pagination.js b/packages/analytics/src/models/pagination.js index d253bec..255d9fd 100644 --- a/packages/analytics/src/models/pagination.js +++ b/packages/analytics/src/models/pagination.js @@ -14,6 +14,10 @@ class Asc { getOrder() { return 1 } + + getName() { + return ASC + } } class Desc { @@ -24,6 +28,10 @@ class Desc { getOrder() { return -1 } + + getName() { + return DESC + } } export default class Pagination { @@ -59,4 +67,8 @@ export default class Pagination { limit() { return { $limit: this.itemsPerPage } } + + getOrder() { + return this.order.getName() + } } diff --git a/packages/analytics/tests/pagination.test.js b/packages/analytics/tests/pagination.test.js index 9c1db98..6d138d1 100644 --- a/packages/analytics/tests/pagination.test.js +++ b/packages/analytics/tests/pagination.test.js @@ -3,7 +3,8 @@ import Pagination, { ASC, DESC } from '../src/models/pagination.js' describe('Pagination', () => { it('should failed when not provide order', () => { - expect(() => new Pagination()).toThrow('order must be asc or desc') + expect(new Pagination('').getOrder()).toBe(DESC) + expect(() => new Pagination('Hahaha')).toThrow() }) it('should return correct limit', () => { const pagination = new Pagination(DESC, 10) diff --git a/packages/analytics/views/commands.handlebars b/packages/analytics/views/commands.handlebars index 1d7034f..f2252cd 100644 --- a/packages/analytics/views/commands.handlebars +++ b/packages/analytics/views/commands.handlebars @@ -1,6 +1,10 @@

Commands

Error commands

+ @@ -21,7 +25,7 @@ {{/each}}
Command
-
- \ No newline at end of file diff --git a/packages/analytics/views/sessions.handlebars b/packages/analytics/views/sessions.handlebars index 236795f..613435f 100644 --- a/packages/analytics/views/sessions.handlebars +++ b/packages/analytics/views/sessions.handlebars @@ -1,4 +1,8 @@

Sessions

+
@@ -26,5 +30,5 @@
\ No newline at end of file diff --git a/packages/analytics/views/users.handlebars b/packages/analytics/views/users.handlebars index 8cb407f..dde1654 100644 --- a/packages/analytics/views/users.handlebars +++ b/packages/analytics/views/users.handlebars @@ -1,4 +1,8 @@

Users List

+
@@ -22,5 +26,5 @@
\ No newline at end of file From b0e12b2027cb670aa3fcbcdbe08fad0aec085f81 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 17:58:52 +0800 Subject: [PATCH 17/19] refactor: use questList to calculate total quests and completed --- packages/analytics/src/controllers/index.js | 15 ++++-- packages/analytics/src/models/quests.js | 55 --------------------- 2 files changed, 12 insertions(+), 58 deletions(-) diff --git a/packages/analytics/src/controllers/index.js b/packages/analytics/src/controllers/index.js index 8557cba..19edb1d 100644 --- a/packages/analytics/src/controllers/index.js +++ b/packages/analytics/src/controllers/index.js @@ -1,13 +1,22 @@ import { userCount } from '../models/users.js' -import { totalQuests, totalCompleted } from '../models/quests.js' +import { questList } from '../models/quests.js' // eslint-disable-next-line import/prefer-default-export export async function homeController(req, res) { + const quests = await questList() + const [totalQuests, totalCompleted] = quests.reduce( + ([total, completed], quest) => [ + total + quest.started, + completed + quest.completed, + ], + [0, 0] + ) + const payload = { title: 'Linux Odyssey Admin', userCount: await userCount(), - totalQuests: await totalQuests(), - totalCompleted: await totalCompleted(), + totalQuests, + totalCompleted, } payload.completedRate = Math.round((payload.totalCompleted / payload.totalQuests) * 10000) / 100 diff --git a/packages/analytics/src/models/quests.js b/packages/analytics/src/models/quests.js index b5f7471..2b64889 100644 --- a/packages/analytics/src/models/quests.js +++ b/packages/analytics/src/models/quests.js @@ -2,61 +2,6 @@ /* eslint-disable func-names */ import { UserProfile } from '@linux-odyssey/models' -export function totalQuests() { - return UserProfile.aggregate([ - { - $project: { - _id: 0, - progress: { $objectToArray: '$progress' }, - }, - }, - { - $unwind: '$progress', - }, - { - $group: { - _id: null, - totalCount: { $sum: 1 }, - }, - }, - ]).then((results) => { - if (results.length === 0) { - throw new Error('No documents found') - } - return results[0].totalCount - }) -} - -export function totalCompleted() { - return UserProfile.aggregate([ - { - $project: { - _id: 0, - progress: { $objectToArray: '$progress' }, - }, - }, - { - $unwind: '$progress', - }, - { - $match: { - 'progress.v.completed': true, - }, - }, - { - $group: { - _id: null, - totalCount: { $sum: 1 }, - }, - }, - ]).then((results) => { - if (results.length === 0) { - throw new Error('No documents found') - } - return results[0].totalCount - }) -} - export function questList() { return UserProfile.aggregate([ { From b96bcb27ad1312490d636d428f6787be0b090a89 Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 18:01:25 +0800 Subject: [PATCH 18/19] refactor: remove unused functions --- packages/analytics/src/models/sessions.js | 23 ----------------------- packages/analytics/src/models/users.js | 1 - 2 files changed, 24 deletions(-) diff --git a/packages/analytics/src/models/sessions.js b/packages/analytics/src/models/sessions.js index 7bc7022..f7b4485 100644 --- a/packages/analytics/src/models/sessions.js +++ b/packages/analytics/src/models/sessions.js @@ -1,28 +1,5 @@ import { Command, Session } from '@linux-odyssey/models' -export function sessionCount() { - return Session.find().count() -} - -export function sessionCompleted() { - return Session.find({ status: 'finished' }).count() -} - -export function averageTimeUsage() { - return Session.aggregate([ - { - $group: { - _id: '$quest', - average: { - $avg: { - $subtract: ['$finishedAt', '$createdAt'], - }, - }, - }, - }, - ]) -} - function formatTime(time) { const seconds = Math.floor(time / 1000) const minutes = Math.floor(seconds / 60) diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index 2452123..896fe5d 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -1,4 +1,3 @@ -import mongoose from 'mongoose' import { Session, User } from '@linux-odyssey/models' function loginMethods(user) { From 9c650a4c26addb238043189f766adb555af2ec7d Mon Sep 17 00:00:00 2001 From: "Chen, Tai-Lin" Date: Thu, 21 Dec 2023 18:09:48 +0800 Subject: [PATCH 19/19] fix: restore db name --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ef5ca27..13285d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: - ./logs:/app/logs environment: - BASE_URL=http://localhost:5173 - - MONGO_URL=mongodb://db:27017/linux-odyssey + - MONGO_URL=mongodb://db:27017/odyssey-test - DOCKER_NETWORK=linux-odyssey-players - SECRET_KEY=b1d2e917-17b9-4fce-9055-082dee6ec5d6 # Only for testing - TRUSTED_PROXIES=${TRUSTED_PROXIES} @@ -64,7 +64,7 @@ services: ports: - '3001:3001' environment: - - MONGO_URL=mongodb://db:27017/linux-odyssey + - MONGO_URL=mongodb://db:27017/odyssey-test - ADMIN_USERNAME=alex - ADMIN_PASSWORD=Alex1234 volumes: