From 81581c4635aafcd1e19cd1ec0b581e47f1c54b32 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 16:29:37 +0800 Subject: [PATCH 01/14] BREAKING: change user social login schema --- packages/models/user.js | 27 +++++++++++++++----- server/src/api/controllers/authController.js | 12 +++++---- server/src/api/routes/authRouter.js | 6 ++--- server/src/auth/githubStrategy.js | 2 +- server/src/auth/googleStrategy.js | 2 +- server/src/auth/oauthVerify.js | 13 +++++----- server/src/auth/passport.js | 18 ++++++++----- server/src/models/userManager.js | 11 ++++++-- 8 files changed, 59 insertions(+), 32 deletions(-) diff --git a/packages/models/user.js b/packages/models/user.js index 6c3bc44..1fad86d 100644 --- a/packages/models/user.js +++ b/packages/models/user.js @@ -1,5 +1,20 @@ import { model, Schema } from 'mongoose' +const socialLoginSchema = new Schema({ + provider: { + type: String, + required: true, + }, + id: { + type: String, + required: true, + }, + displayName: { + type: String, + required: false, + }, +}) + const User = model( 'User', new Schema( @@ -15,13 +30,11 @@ const User = model( required: true, }, hashedPassword: String, - google: { - id: String, - displayName: String, - }, - github: { - id: String, - displayName: String, + socialLogins: { + type: Map, + of: socialLoginSchema, + default: {}, + required: true, }, }, { diff --git a/server/src/api/controllers/authController.js b/server/src/api/controllers/authController.js index 8ae7978..30af2c1 100644 --- a/server/src/api/controllers/authController.js +++ b/server/src/api/controllers/authController.js @@ -1,5 +1,4 @@ import { matchedData } from 'express-validator' -import { User } from '@linux-odyssey/models' import { genJWT } from '../../utils/auth.js' import { createUser } from '../../models/userManager.js' import { asyncHandler } from '../../middleware/error.js' @@ -50,7 +49,7 @@ export function checkSession(req, res) { }) } -export function socialLogin(req, res) { +export function socialLoginHandler(req, res) { const { newUser } = req.user if (newUser) { req.session.newUser = newUser @@ -71,10 +70,13 @@ export const registerFromSession = asyncHandler(async (req, res, next) => { return } - newUser.username = username - const user = new User(newUser) + const { email, socialLogin } = newUser - await user.save() + if (!socialLogin) { + throw new Error('Socail Login not found') + } + + const user = await createUser(username, email, { socialLogin }) req.login(user, (err) => { if (err) { diff --git a/server/src/api/routes/authRouter.js b/server/src/api/routes/authRouter.js index 6307264..2aa0319 100644 --- a/server/src/api/routes/authRouter.js +++ b/server/src/api/routes/authRouter.js @@ -7,8 +7,8 @@ import { register, checkSession, logout, - socialLogin, registerFromSession, + socialLoginHandler, } from '../controllers/authController.js' import { checkUsernameValidators, @@ -49,7 +49,7 @@ if (enabledMethods.google) { '/google/callback', authenticateRateLimit, passport.authenticate('google'), - socialLogin + socialLoginHandler ) } @@ -59,7 +59,7 @@ if (enabledMethods.github) { '/github/callback', authenticateRateLimit, passport.authenticate('github'), - socialLogin + socialLoginHandler ) } diff --git a/server/src/auth/githubStrategy.js b/server/src/auth/githubStrategy.js index d3af989..e6fe4ce 100644 --- a/server/src/auth/githubStrategy.js +++ b/server/src/auth/githubStrategy.js @@ -4,7 +4,7 @@ import config from '../config.js' import oauthVerify from './oauthVerify.js' function verify(accessToken, refreshToken, profile, cb) { - oauthVerify('github', profile, { 'github.id': profile.id }, cb) + oauthVerify('github', profile, { 'socialLogins.github.id': profile.id }, cb) } const enabled = config.github.clientID && config.github.clientSecret diff --git a/server/src/auth/googleStrategy.js b/server/src/auth/googleStrategy.js index 566a09e..a29b79d 100644 --- a/server/src/auth/googleStrategy.js +++ b/server/src/auth/googleStrategy.js @@ -4,7 +4,7 @@ import config from '../config.js' import oauthVerify from './oauthVerify.js' function verify(issuer, profile, cb) { - oauthVerify('google', profile, { 'google.id': profile.id }, cb) + oauthVerify('google', profile, { 'socialLogins.google.id': profile.id }, cb) } const enabled = config.google.clientID && config.google.clientSecret diff --git a/server/src/auth/oauthVerify.js b/server/src/auth/oauthVerify.js index 409fde3..8699ca8 100644 --- a/server/src/auth/oauthVerify.js +++ b/server/src/auth/oauthVerify.js @@ -7,18 +7,19 @@ export default async function oauthVerify(provider, profile, query, cb) { return cb(new Error('No email found')) } + const socialLogin = { + provider, + id: profile.id, + displayName: profile.displayName, + } let user = await User.findOne(query) if (!user) { user = await User.findOne({ email }) if (!user) { - const newUser = { email } - newUser[provider] = profile + const newUser = { email, socialLogin } return cb(null, { newUser }) } - user[provider] = { - id: profile.id, - displayName: profile.displayName, - } + user.socialLogins.set(provider, socialLogin) await user.save() } diff --git a/server/src/auth/passport.js b/server/src/auth/passport.js index 47f9776..16b26f9 100644 --- a/server/src/auth/passport.js +++ b/server/src/auth/passport.js @@ -26,14 +26,18 @@ if (enabledMethods.github) { passport.serializeUser((user, done) => { process.nextTick(async () => { - if (!(await UserProfile.exists({ user: user._id }))) { - UserProfile.create({ user: user._id }) + try { + if (user._id && !(await UserProfile.exists({ user: user._id }))) { + await UserProfile.create({ user: user._id }) + } + done(null, { + _id: user._id, + username: user.username, + email: user.email, + }) + } catch (err) { + done(err) } - done(null, { - _id: user._id, - username: user.username, - email: user.email, - }) }) }) diff --git a/server/src/models/userManager.js b/server/src/models/userManager.js index aa877fb..1fa1fe6 100644 --- a/server/src/models/userManager.js +++ b/server/src/models/userManager.js @@ -1,16 +1,23 @@ import { User, UserProfile } from '@linux-odyssey/models' import { hashPassword } from '../utils/auth.js' -export async function createUser(username, email, password) { +export async function createUser(username, email, { password, socialLogin }) { const user = new User({ username, email, }) - user.hashedPassword = await hashPassword(password) + if (password) { + user.hashedPassword = await hashPassword(password) + } + if (socialLogin) { + user.socialLogins.set(socialLogin.provider, socialLogin) + } await user.save() const userProfile = new UserProfile({ user, }) await userProfile.save() + + return user } From 7d25c7efaa9e9a8dca1072e3f7e83237d7a3bca1 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 18:05:42 +0800 Subject: [PATCH 02/14] feat: update workflows * build images with automatic tags * separate staging and production deployment * reuse workflows --- .github/workflows/build-docker-image.yml | 51 ++++++++ .github/workflows/build.yml | 32 +++++ .github/workflows/ci-checks.yml | 45 +++++++ .github/workflows/ci_cd.yml | 110 ------------------ .github/workflows/production.yml | 27 +++++ .../{code_review.yml => pull-request.yml} | 14 ++- .github/workflows/staging.yml | 27 +++++ docker-compose.prod.yml | 9 +- 8 files changed, 197 insertions(+), 118 deletions(-) create mode 100644 .github/workflows/build-docker-image.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/ci-checks.yml delete mode 100644 .github/workflows/ci_cd.yml create mode 100644 .github/workflows/production.yml rename .github/workflows/{code_review.yml => pull-request.yml} (64%) create mode 100644 .github/workflows/staging.yml diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml new file mode 100644 index 0000000..8ad8529 --- /dev/null +++ b/.github/workflows/build-docker-image.yml @@ -0,0 +1,51 @@ +name: Build Docker Image + +on: + workflow_call: + inputs: + imageName: + type: string + required: true + description: 'Name of the image to build' + dockerfilePath: + type: string + required: true + description: 'Path to the Dockerfile' + target: + type: string + required: false + description: 'Target to build' + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ inputs.imageName }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build Image + uses: docker/build-push-action@v2 + with: + context: . + file: ${{ inputs.dockerfilePath }} + target: ${{ inputs.target }} + push: true + tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..a95029d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,32 @@ +name: Build + +on: + workflow_call: + +jobs: + build-backend: + uses: ./.github/workflows/build-docker-image.yml + with: + imageName: 'linuxodyssey/backend' + dockerfilePath: './server/Dockerfile' + target: prod + + build-frontend: + uses: ./.github/workflows/build-docker-image.yml + with: + imageName: 'linuxodyssey/frontend' + dockerfilePath: './app/Dockerfile' + target: prod + + build-analytics: + uses: ./.github/workflows/build-docker-image.yml + with: + imageName: 'linuxodyssey/analytics' + dockerfilePath: './packages/analytics/Dockerfile' + target: prod + + build-base: + uses: ./.github/workflows/build-docker-image.yml + with: + imageName: 'linuxodyssey/base' + dockerfilePath: './quests/Dockerfile' diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml new file mode 100644 index 0000000..2d7bb93 --- /dev/null +++ b/.github/workflows/ci-checks.yml @@ -0,0 +1,45 @@ +name: CI Checks + +on: + workflow_call: + +jobs: + ci: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: Install dependencies + run: yarn install + + - name: Lint + run: yarn lint + + - name: Test + run: yarn test + + e2e: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Build Test Environment, Stage 1 + run: docker-compose -f docker-compose.testing.yml build base backend app-test + + - name: Build Test Quests + run: docker-compose -f docker-compose.testing.yml run --rm backend yarn build:quests + + - name: Run Cypress Tests + env: + GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} + GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} + GITHUB_CLIENT_ID: ${{ secrets.GITHUB_CLIENT_ID }} + GITHUB_CLIENT_SECRET: ${{ secrets.GITHUB_CLIENT_SECRET }} + run: docker-compose -f docker-compose.testing.yml run cypress diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml deleted file mode 100644 index e90c3bb..0000000 --- a/.github/workflows/ci_cd.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: CI/CD - -on: - push: - -jobs: - ci: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - uses: actions/setup-node@v3 - with: - node-version: 20 - - - name: Install dependencies - run: yarn install - - - name: Lint - run: yarn lint - - - name: Test - run: yarn test - - e2e: - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Build Test Environment, Stage 1 - run: docker-compose -f docker-compose.testing.yml build base backend app-test - - - name: Build Test Quests - run: docker-compose -f docker-compose.testing.yml run --rm backend yarn build:quests - - - name: Run Cypress Tests - env: - GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} - GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} - GITHUB_CLIENT_ID: ${{ secrets.GITHUB_CLIENT_ID }} - GITHUB_CLIENT_SECRET: ${{ secrets.GITHUB_CLIENT_SECRET }} - run: docker-compose -f docker-compose.testing.yml run cypress - - build-and-push: - needs: - - ci - - e2e - if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build backend - uses: docker/build-push-action@v2 - with: - context: . - file: ./server/Dockerfile - target: prod - push: true - tags: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/linux-odyssey:backend - - - name: Build app - uses: docker/build-push-action@v2 - with: - context: . - file: ./app/Dockerfile - target: prod - push: true - tags: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/linux-odyssey:app - - - name: Build analytics - uses: docker/build-push-action@v2 - with: - context: . - file: ./packages/analytics/Dockerfile - target: prod - push: true - tags: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/linux-odyssey:analytics - - - name: Build base image - uses: docker/build-push-action@v2 - with: - context: . - file: ./quests/Dockerfile - push: true - tags: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/linux-odyssey:base - - deploy: - needs: build-and-push - runs-on: ubuntu-latest - - steps: - - name: SSH into server and update service - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.SERVER_HOST }} - port: ${{ secrets.SERVER_PORT }} - username: ${{ secrets.SERVER_USER }} - key: ${{ secrets.SERVER_SSH_KEY }} diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml new file mode 100644 index 0000000..e82c2f0 --- /dev/null +++ b/.github/workflows/production.yml @@ -0,0 +1,27 @@ +name: Production Deployment + +on: + push: + tags: + - 'v*' + +jobs: + ci-checks: + uses: ./.github/workflows/ci-checks.yml + + build: + needs: ci-checks + uses: ./.github/workflows/build.yml + + deploy: + needs: build + runs-on: ubuntu-latest + + steps: + - name: SSH into server and update service + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.SERVER_HOST }} + port: ${{ secrets.SERVER_PORT }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} diff --git a/.github/workflows/code_review.yml b/.github/workflows/pull-request.yml similarity index 64% rename from .github/workflows/code_review.yml rename to .github/workflows/pull-request.yml index e15258e..56a6180 100644 --- a/.github/workflows/code_review.yml +++ b/.github/workflows/pull-request.yml @@ -1,11 +1,13 @@ -name: Code Review GPT +name: Pull Request Workflow on: pull_request: - branches: [main] jobs: - run_code_review: + ci-checks: + uses: ./.github/workflows/ci-checks.yml + + code_review: permissions: pull-requests: write runs-on: ubuntu-latest @@ -18,5 +20,9 @@ jobs: uses: mattzcarey/code-review-gpt@v0.1.4-alpha with: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - MODEL: 'gpt-3.5-turbo' + MODEL: 'gpt-4-1106-preview' GITHUB_TOKEN: ${{ github.token }} + + build: + uses: ./.github/workflows/build.yml + needs: ci-checks diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml new file mode 100644 index 0000000..085090a --- /dev/null +++ b/.github/workflows/staging.yml @@ -0,0 +1,27 @@ +name: Staging Deployment + +on: + push: + branches: + - main + +jobs: + ci-checks: + uses: ./.github/workflows/ci-checks.yml + + build: + uses: ./.github/workflows/build.yml + needs: ci-checks + + deploy: + needs: build + runs-on: ubuntu-latest + + steps: + - name: SSH into server and update service + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.SERVER_HOST }} + port: ${{ secrets.SERVER_PORT }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 23c00ac..d5b0eed 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,7 +1,7 @@ version: '3' services: app: - image: lancatlin/linux-odyssey:app + image: linuxodyssey/frontend depends_on: - backend restart: always @@ -13,8 +13,9 @@ services: internal: ports: - 8080:80 + backend: - image: lancatlin/linux-odyssey:backend + image: linuxodyssey/backend depends_on: - db restart: always @@ -36,7 +37,7 @@ services: - TRUSTED_PROXIES=${TRUSTED_PROXIES} analytics: - image: lancatlin/linux-odyssey:analytics + image: linuxodyssey/analytics build: context: . dockerfile: packages/analytics/Dockerfile @@ -76,7 +77,7 @@ services: - BUG_REPORT_URL=${BUG_REPORT_URL} base: - image: lancatlin/linux-odyssey:base + image: linuxodyssey/base build: context: . dockerfile: ./quests/Dockerfile From d3945eeaa7a9dce4b780c43fd9e9ab151dbeda35 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 18:12:03 +0800 Subject: [PATCH 03/14] fix: change code review model to gpt 4 --- .github/workflows/pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 56a6180..4ac6f97 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -20,7 +20,7 @@ jobs: uses: mattzcarey/code-review-gpt@v0.1.4-alpha with: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - MODEL: 'gpt-4-1106-preview' + MODEL: 'gpt-4' GITHUB_TOKEN: ${{ github.token }} build: From f81c96be62931a9a6002dc59490d0fa5347ce08b Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 18:22:27 +0800 Subject: [PATCH 04/14] fix: fix createUser without password --- server/src/api/controllers/authController.js | 2 +- server/src/models/userManager.js | 5 +++-- server/src/utils/setupTest.js | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/api/controllers/authController.js b/server/src/api/controllers/authController.js index 30af2c1..2ddcd50 100644 --- a/server/src/api/controllers/authController.js +++ b/server/src/api/controllers/authController.js @@ -16,7 +16,7 @@ export const issueToken = asyncHandler(async (req, res) => { export const register = asyncHandler(async (req, res, next) => { const { username, password, email } = matchedData(req) - const user = await createUser(username, email, password) + const user = await createUser(username, email, { password }) req.login(user, (err) => { if (err) { next(err) diff --git a/server/src/models/userManager.js b/server/src/models/userManager.js index 1fa1fe6..46943dd 100644 --- a/server/src/models/userManager.js +++ b/server/src/models/userManager.js @@ -8,9 +8,10 @@ export async function createUser(username, email, { password, socialLogin }) { }) if (password) { user.hashedPassword = await hashPassword(password) - } - if (socialLogin) { + } else if (socialLogin) { user.socialLogins.set(socialLogin.provider, socialLogin) + } else { + throw new Error('No password or social login provided') } await user.save() diff --git a/server/src/utils/setupTest.js b/server/src/utils/setupTest.js index 2c97278..5369d4c 100644 --- a/server/src/utils/setupTest.js +++ b/server/src/utils/setupTest.js @@ -17,6 +17,6 @@ export default async function setupTest() { return } - await createUser(username, email, password) + await createUser(username, email, { password }) console.log('Created testing account') } From 17277da171cc9205af82a70d68946791b4c4ecb7 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 18:22:47 +0800 Subject: [PATCH 05/14] fix: build cypress along with other images --- .github/workflows/ci-checks.yml | 2 +- docker-compose.testing.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index 2d7bb93..4ed25a8 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@v3 - name: Build Test Environment, Stage 1 - run: docker-compose -f docker-compose.testing.yml build base backend app-test + run: docker-compose -f docker-compose.testing.yml build base backend app-test cypress - name: Build Test Quests run: docker-compose -f docker-compose.testing.yml run --rm backend yarn build:quests diff --git a/docker-compose.testing.yml b/docker-compose.testing.yml index 1fdf51d..dada79f 100644 --- a/docker-compose.testing.yml +++ b/docker-compose.testing.yml @@ -69,6 +69,8 @@ services: - MONGO_USER=user - MONGO_PASS=user-password - MONGO_DB=odyssey-test + ports: + - 27017:27017 base: image: lancatlin/linux-odyssey:base From bf6aabb1c596318066692d4919f826f4140a2d39 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 18:37:29 +0800 Subject: [PATCH 06/14] fix: passing docker hub secrets to build workflow --- .github/workflows/build-docker-image.yml | 6 ++++++ .github/workflows/build.yml | 4 ++++ .github/workflows/production.yml | 1 + .github/workflows/pull-request.yml | 3 ++- .github/workflows/staging.yml | 3 ++- 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 8ad8529..8b0716c 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -16,6 +16,12 @@ on: required: false description: 'Target to build' + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true + jobs: build-and-push: runs-on: ubuntu-latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a95029d..222368f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,7 @@ jobs: imageName: 'linuxodyssey/backend' dockerfilePath: './server/Dockerfile' target: prod + secrets: inherit build-frontend: uses: ./.github/workflows/build-docker-image.yml @@ -17,6 +18,7 @@ jobs: imageName: 'linuxodyssey/frontend' dockerfilePath: './app/Dockerfile' target: prod + secrets: inherit build-analytics: uses: ./.github/workflows/build-docker-image.yml @@ -24,9 +26,11 @@ jobs: imageName: 'linuxodyssey/analytics' dockerfilePath: './packages/analytics/Dockerfile' target: prod + secrets: inherit build-base: uses: ./.github/workflows/build-docker-image.yml with: imageName: 'linuxodyssey/base' dockerfilePath: './quests/Dockerfile' + secrets: inherit diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index e82c2f0..6db9897 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -12,6 +12,7 @@ jobs: build: needs: ci-checks uses: ./.github/workflows/build.yml + secrets: inherit deploy: needs: build diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 4ac6f97..e93272f 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -24,5 +24,6 @@ jobs: GITHUB_TOKEN: ${{ github.token }} build: - uses: ./.github/workflows/build.yml needs: ci-checks + uses: ./.github/workflows/build.yml + secrets: inherit diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 085090a..48e761c 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -10,8 +10,9 @@ jobs: uses: ./.github/workflows/ci-checks.yml build: - uses: ./.github/workflows/build.yml needs: ci-checks + uses: ./.github/workflows/build.yml + secrets: inherit deploy: needs: build From 8d0ec005363a7f384fa86fc3f93ea50ffe75b377 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 18:43:26 +0800 Subject: [PATCH 07/14] fix: checkout code on building --- .github/workflows/build-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 8b0716c..e56ae14 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -27,8 +27,8 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Checkout + uses: actions/checkout@v3 - name: Docker meta id: meta From 8430ce442857b1f648bdcdde44ace8bfeb0fe7ad Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 21:46:03 +0800 Subject: [PATCH 08/14] feat: add error handling for oauthVerify --- server/src/auth/oauthVerify.js | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/server/src/auth/oauthVerify.js b/server/src/auth/oauthVerify.js index 8699ca8..ea4aedd 100644 --- a/server/src/auth/oauthVerify.js +++ b/server/src/auth/oauthVerify.js @@ -2,27 +2,31 @@ import { User } from '@linux-odyssey/models' export default async function oauthVerify(provider, profile, query, cb) { process.nextTick(async () => { - const email = profile.emails[0]?.value - if (!email) { - return cb(new Error('No email found')) - } - - const socialLogin = { - provider, - id: profile.id, - displayName: profile.displayName, - } - let user = await User.findOne(query) - if (!user) { - user = await User.findOne({ email }) - if (!user) { - const newUser = { email, socialLogin } - return cb(null, { newUser }) + try { + const email = profile.emails[0]?.value + if (!email) { + return cb(new Error('No email found')) } - user.socialLogins.set(provider, socialLogin) - await user.save() - } - return cb(null, user) + const socialLogin = { + provider, + id: profile.id, + displayName: profile.displayName, + } + let user = await User.findOne(query) + if (!user) { + user = await User.findOne({ email }) + if (!user) { + const newUser = { email, socialLogin } + return cb(null, { newUser }) + } + user.socialLogins.set(provider, socialLogin) + await user.save() + } + + return cb(null, user) + } catch (err) { + return cb(err) + } }) } From 1ab394f1872c0b9aa0922d91d4ca1137cb939778 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 22:22:34 +0800 Subject: [PATCH 09/14] feat: add basic auth to analytics --- packages/analytics/package.json | 1 + packages/analytics/src/config.js | 9 +++++++++ packages/analytics/src/index.js | 8 ++++++++ yarn.lock | 19 +++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/packages/analytics/package.json b/packages/analytics/package.json index 53eda5a..d035121 100644 --- a/packages/analytics/package.json +++ b/packages/analytics/package.json @@ -12,6 +12,7 @@ "dependencies": { "@linux-odyssey/models": "*", "express": "^4.18.2", + "express-basic-auth": "^1.2.1", "express-handlebars": "^7.1.2" }, "devDependencies": { diff --git a/packages/analytics/src/config.js b/packages/analytics/src/config.js index 00e1afd..a50eb7e 100644 --- a/packages/analytics/src/config.js +++ b/packages/analytics/src/config.js @@ -1,7 +1,16 @@ +function getOrFailed(key) { + if (!process.env[key]) { + throw new Error(`Missing required environment variable: ${key}`) + } + return process.env[key] +} + const config = { host: process.env.HOST || 'localhost', port: process.env.PORT || 3001, db: process.env.MONGO_URL || 'mongodb://localhost:27017/odyssey-test', + username: getOrFailed('USERNAME'), + password: getOrFailed('PASSWORD'), } config.baseUrl = process.env.BASE_URL || `http://${config.host}:${config.port}` diff --git a/packages/analytics/src/index.js b/packages/analytics/src/index.js index 6977ab0..581a28e 100644 --- a/packages/analytics/src/index.js +++ b/packages/analytics/src/index.js @@ -1,6 +1,7 @@ import connectDB from '@linux-odyssey/models' import express from 'express' import { engine } from 'express-handlebars' +import basicAuth from 'express-basic-auth' import config from './config.js' import router from './router.js' @@ -13,6 +14,13 @@ async function main() { app.set('view engine', 'handlebars') app.set('views', './views') + app.use( + basicAuth({ + users: { [config.username]: config.password }, + challenge: true, + realm: 'Linux Odyssey Admin Dashboard', + }) + ) app.use(router) app.use(express.static('./public')) diff --git a/yarn.lock b/yarn.lock index 19a6782..bd4e3ec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1593,6 +1593,13 @@ base64url@3.x.x: resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d" integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A== +basic-auth@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a" + integrity sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg== + dependencies: + safe-buffer "5.1.2" + bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" @@ -2850,6 +2857,13 @@ expect@^29.6.1: jest-message-util "^29.6.1" jest-util "^29.6.1" +express-basic-auth@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/express-basic-auth/-/express-basic-auth-1.2.1.tgz#d31241c03a915dd55db7e5285573049cfcc36381" + integrity sha512-L6YQ1wQ/mNjVLAmK3AG1RK6VkokA1BIY6wmiH304Xtt/cLTps40EusZsU1Uop+v9lTDPxdtzbFmdXfFO3KEnwA== + dependencies: + basic-auth "^2.0.1" + express-handlebars@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/express-handlebars/-/express-handlebars-7.1.2.tgz#2471673d11af46f496cba4098a705f0217232fda" @@ -5581,6 +5595,11 @@ rxjs@^7.5.1: dependencies: tslib "^2.1.0" +safe-buffer@5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" From aed66a4fe5e677b27ef02f7a6eb5f12c5ce80682 Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 22:22:51 +0800 Subject: [PATCH 10/14] fix: analytics login method --- packages/analytics/src/models/users.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/analytics/src/models/users.js b/packages/analytics/src/models/users.js index ae1aee6..69d8310 100644 --- a/packages/analytics/src/models/users.js +++ b/packages/analytics/src/models/users.js @@ -3,8 +3,7 @@ import { Session, User } from '@linux-odyssey/models' function loginMethods(user) { const methods = [] if (user.hashedPassword) methods.push('password') - if (user.google?.id) methods.push('google') - if (user.github?.id) methods.push('github') + if (user.socialLogins) methods.push(...Object.keys(user.socialLogins)) return methods } From f9e0fade3b9fa5a349a09d761563d95edb9f482c Mon Sep 17 00:00:00 2001 From: wancat Date: Sat, 18 Nov 2023 22:25:05 +0800 Subject: [PATCH 11/14] fix: update quest image name --- .github/workflows/build.yml | 2 +- docker-compose.prod.yml | 2 +- docker-compose.testing.yml | 6 +++--- docker-compose.yml | 5 ++++- quests/helloworld/Dockerfile | 2 +- quests/spell/Dockerfile | 2 +- server/src/config.js | 13 ++++++++++--- server/src/containers/docker.js | 10 +++++----- 8 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 222368f..7a9bf3e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,6 @@ jobs: build-base: uses: ./.github/workflows/build-docker-image.yml with: - imageName: 'linuxodyssey/base' + imageName: 'linuxodyssey/quest-base' dockerfilePath: './quests/Dockerfile' secrets: inherit diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index d5b0eed..ef05c98 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -77,7 +77,7 @@ services: - BUG_REPORT_URL=${BUG_REPORT_URL} base: - image: linuxodyssey/base + image: linuxodyssey/quest-base build: context: . dockerfile: ./quests/Dockerfile diff --git a/docker-compose.testing.yml b/docker-compose.testing.yml index dada79f..683b22f 100644 --- a/docker-compose.testing.yml +++ b/docker-compose.testing.yml @@ -16,7 +16,7 @@ services: - app-test app-test: - image: lancatlin/linux-odyssey:app + image: linuxodyssey/app build: context: . dockerfile: app/Dockerfile @@ -29,7 +29,7 @@ services: - backend backend: - image: lancatlin/linux-odyssey:backend + image: linuxodyssey/backend build: context: . dockerfile: server/Dockerfile @@ -73,7 +73,7 @@ services: - 27017:27017 base: - image: lancatlin/linux-odyssey:base + image: linuxodyssey/quest-base build: context: . dockerfile: ./quests/Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index c991351..3aeaf0b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: volumes: - ./app:/app/app - ./packages:/app/packages + backend: depends_on: - db @@ -62,6 +63,8 @@ services: - '3001:3001' environment: - MONGO_URL=mongodb://db:27017/odyssey-test + - USERNAME=user + - PASSWORD=user volumes: - ./packages/analytics:/app/packages/analytics - ./packages/models:/app/packages/models @@ -85,7 +88,7 @@ services: - SWAGGER_FILE=/tmp/swagger.yaml base: - image: lancatlin/linux-odyssey:base + image: linuxodyssey/quest-base build: context: . dockerfile: ./quests/Dockerfile diff --git a/quests/helloworld/Dockerfile b/quests/helloworld/Dockerfile index fe63a59..1811d6e 100644 --- a/quests/helloworld/Dockerfile +++ b/quests/helloworld/Dockerfile @@ -1,3 +1,3 @@ -FROM lancatlin/linux-odyssey:base +FROM linuxodyssey/quest-base COPY home ${HOME} diff --git a/quests/spell/Dockerfile b/quests/spell/Dockerfile index fe63a59..1811d6e 100644 --- a/quests/spell/Dockerfile +++ b/quests/spell/Dockerfile @@ -1,3 +1,3 @@ -FROM lancatlin/linux-odyssey:base +FROM linuxodyssey/quest-base COPY home ${HOME} diff --git a/server/src/config.js b/server/src/config.js index eb0b7b9..65008f8 100644 --- a/server/src/config.js +++ b/server/src/config.js @@ -24,7 +24,6 @@ const config = { port: get('PORT', 3000), db: get('MONGO_URL', 'mongodb://localhost:27017/odyssey-test'), secret: get('SECRET_KEY', ''), - dockerNetwork: get('DOCKER_NETWORK', 'linux-odyssey-players'), isProduction: process.env.NODE_ENV === 'production', hostPwd: get('HOST_PWD', ''), containerExpiry: get('EXPIRY', 1000 * 60 * 60), @@ -40,9 +39,13 @@ const config = { trustedProxies: getTrustProxies('TRUSTED_PROXIES'), surveyUrl: getUrl('SURVEY_URL', ''), bugReportUrl: getUrl('BUG_REPORT_URL', ''), - dockerImage: get('DOCKER_IMAGE', 'lancatlin/linux-odyssey'), - mountQuest: get('MOUNT_QUEST', ''), + docker: { + network: get('DOCKER_NETWORK', 'linux-odyssey-players'), + defaultImage: get('QUEST_IMAGE', 'linuxodyssey/quest-base'), + imagePrefix: get('DOCKER_PREFIX', 'linuxodyssey/quest-'), + mountQuest: get('MOUNT_QUEST', ''), + }, } config.baseUrl = get('BASE_URL', `http://${config.host}:${config.port}`) @@ -52,4 +55,8 @@ config.testing = { password: get('TESTING_PASSWORD', ''), } +export function getQuestImage(id) { + return `${config.docker.imagePrefix}${id}` +} + export default config diff --git a/server/src/containers/docker.js b/server/src/containers/docker.js index f7343fd..e71337f 100644 --- a/server/src/containers/docker.js +++ b/server/src/containers/docker.js @@ -1,5 +1,5 @@ import Docker from 'dockerode' -import config from '../config.js' +import config, { getQuestImage } from '../config.js' const engine = new Docker() @@ -11,17 +11,17 @@ const containerOptions = { OpenStdin: true, StdinOnce: false, HostConfig: { - NetworkMode: config.dockerNetwork, + NetworkMode: config.docker.network, }, } // const network = engine.getNetwork(config.dockerNetwork) -export function createContainer(name, tag) { +export function createContainer(name, questId) { const option = { ...containerOptions, name, - Image: `${config.dockerImage}:${tag}`, + Image: getQuestImage(questId), } if (!config.isProduction && config.hostPwd && config.mountQuest) { option.HostConfig.Bind = [ @@ -87,7 +87,7 @@ export function buildQuestImage(questPath, questId) { src: ['Dockerfile', 'home'], }, { - t: `${config.dockerImage}:${questId}`, + t: getQuestImage(questId), networkmode: 'none', memory: 10 * 1e6, }, From 77cb555ae6c85b87402f5b9b39616dc5ee5f5fa2 Mon Sep 17 00:00:00 2001 From: wancat Date: Sun, 19 Nov 2023 14:36:56 +0800 Subject: [PATCH 12/14] feat: add production deploy credentials --- .github/workflows/production.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 6db9897..700f295 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -22,7 +22,7 @@ jobs: - name: SSH into server and update service uses: appleboy/ssh-action@master with: - host: ${{ secrets.SERVER_HOST }} - port: ${{ secrets.SERVER_PORT }} - username: ${{ secrets.SERVER_USER }} - key: ${{ secrets.SERVER_SSH_KEY }} + host: ${{ secrets.GCP_SSH_HOST }} + port: ${{ secrets.GCP_SSH_PORT }} + username: ${{ secrets.GCP_SSH_USER }} + key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} From 20280752402b1e88c4324aaa8a6cddd38f85955f Mon Sep 17 00:00:00 2001 From: wancat Date: Sun, 19 Nov 2023 14:51:36 +0800 Subject: [PATCH 13/14] feat: setup admin env in prod compose --- docker-compose.prod.yml | 2 ++ packages/analytics/src/config.js | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index ef05c98..7b9e3e5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -51,6 +51,8 @@ services: - '3001:3001' environment: - MONGO_URL=mongodb://${MONGO_USER}:${MONGO_PASSWORD}@db:27017/${MONGO_DB} + - ADMIN_USERNAME=${ADMIN_USERNAME} + - ADMIN_PASSWORD=${ADMIN_PASSWORD} db: image: mongo diff --git a/packages/analytics/src/config.js b/packages/analytics/src/config.js index a50eb7e..466ea91 100644 --- a/packages/analytics/src/config.js +++ b/packages/analytics/src/config.js @@ -9,8 +9,8 @@ const config = { host: process.env.HOST || 'localhost', port: process.env.PORT || 3001, db: process.env.MONGO_URL || 'mongodb://localhost:27017/odyssey-test', - username: getOrFailed('USERNAME'), - password: getOrFailed('PASSWORD'), + username: getOrFailed('ADMIN_USERNAME'), + password: getOrFailed('ADMIN_PASSWORD'), } config.baseUrl = process.env.BASE_URL || `http://${config.host}:${config.port}` From 521cf46929dc08eda06b4d6dcf261b6027dd968f Mon Sep 17 00:00:00 2001 From: wancat Date: Sun, 19 Nov 2023 14:56:10 +0800 Subject: [PATCH 14/14] feat: add concurrency --- .github/workflows/production.yml | 4 ++++ .github/workflows/pull-request.yml | 4 ++++ .github/workflows/staging.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 700f295..9768d1e 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -5,6 +5,10 @@ on: tags: - 'v*' +concurrency: + group: production + cancel-in-progress: true + jobs: ci-checks: uses: ./.github/workflows/ci-checks.yml diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index e93272f..fb72580 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -3,6 +3,10 @@ name: Pull Request Workflow on: pull_request: +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + jobs: ci-checks: uses: ./.github/workflows/ci-checks.yml diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 48e761c..c2b5818 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -5,6 +5,10 @@ on: branches: - main +concurrency: + group: staging + cancel-in-progress: true + jobs: ci-checks: uses: ./.github/workflows/ci-checks.yml