Feat filegraph (#194)

* feat: update filegraph on touch command

* feat: update filegraph on mkdir

* feat: add different parsing opts for each command

* refactor: replace axios with fetch. put handleCommand to a new file

* feat: add config for mounting cli in dev stage

* refactor: clean up unused validators

* feat: add command handler for copy files. reuse resolvePath function

* feat: add rm handler, support specific files deletion. Glob not supported yet

* feat: handle discover cases that the folder is actually empty

* feat: handle discover empty folder better and support rmdir

* feat: handle mv command

* refactor: remove unused console logs

.eslintrc.js

@@ -84,6 +84,7 @@ module.exports = {
     '@typescript-eslint/no-shadow': 'error',
     'no-restricted-imports': ['error', '../*'],
     'import/no-relative-packages': 'off',
+    'no-await-in-loop': 'off',
   },
   overrides: [
     {

packages/container/cli.js

@@ -1,9 +1,6 @@
-/* eslint-disable no-restricted-syntax */
 const fs = require('fs').promises
 const { exit } = require('process')
-const minimist = require('minimist')
+const handleCommand = require('./handleCommand.js')
-const axios = require('axios')
-const discoverFiles = require('./discover.js')
 
 const {
   API_ENDPOINT,
@@ -15,13 +12,6 @@ const {
   CMD_EXIT_CODE,
 } = process.env
 
-const api = axios.create({
-  baseURL: `${API_ENDPOINT}/api/v1`,
-  headers: {
-    Authorization: `Bearer ${TOKEN}`,
-  },
-})
-
 async function readOrNone(file) {
   try {
     return await fs.readFile(file, 'utf8')
@@ -30,28 +20,6 @@ async function readOrNone(file) {
   }
 }
 
-const commandListeners = {
-  ls: [discoverFiles],
-  cd: [changeDirectory],
-}
-
-function changeDirectory() {
-  return {
-    pwd: process.cwd(),
-  }
-}
-
-async function handleCommand(command) {
-  const argv = minimist(command.split(' '))
-  const name = argv._[0]
-  const listeners = commandListeners[name]
-  if (!listeners) return {}
-  return listeners.reduce(async (result, listener) => {
-    const res = await listener(argv)
-    return { ...res, ...result }
-  }, {})
-}
-
 async function main() {
   if (!CMD_NAME) exit(0)
 
@@ -67,7 +35,23 @@ async function main() {
     pwd: PWD,
     params,
   }
-  await api.post('/commands', payload)
+  try {
+    const response = await fetch(`${API_ENDPOINT}/api/v1/commands`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${TOKEN}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload),
+    })
+    if (!response.ok) {
+      console.error(`Failed to send command: ${response.statusText}`)
+      const responseBody = await response.text()
+      console.error(`Response Body: ${responseBody}`)
+    }
+  } catch (e) {
+    console.error('Error occurred while sending command:', e)
+  }
 }
 
 main()

packages/container/discover.js

@@ -1,21 +1,25 @@
 const fs = require('fs').promises
 const path = require('path')
+const { resolvePath } = require('./utils')
 
 async function collectFilesInfo(inputPath, level = 0, hiddenFiles = false) {
   const stats = await fs.stat(inputPath)
   const name = path.basename(inputPath)
   const discovered = level <= 1 && (hiddenFiles || !name.startsWith('.'))
-  const files = [
+  const self = {
-    {
+    path: inputPath,
-      path: inputPath,
+    name,
-      name,
+    type: stats.isDirectory() ? 'directory' : 'file',
-      type: stats.isDirectory() ? 'directory' : 'file',
+    discovered,
-      discovered,
+  }
-    },
+  const files = [self]
-  ]
   if (stats.isDirectory() && level < 2) {
     try {
       let entries = await fs.readdir(inputPath)
+      if (entries.length === 0) {
+        self.empty = true
+        return files
+      }
       if (level > 0) {
         entries = entries.slice(0, 5)
       }
@@ -37,7 +41,7 @@ async function discoverFiles(argv) {
   try {
     const result = await Promise.all(
       targetPath
-        .map((p) => path.resolve(process.cwd(), p))
+        .map((p) => resolvePath(p))
         .map((p) => collectFilesInfo(p, 0, argv.a || argv.all))
     )
     return { discover: result.flat() }
@@ -46,4 +50,7 @@ async function discoverFiles(argv) {
   }
 }
 
-module.exports = discoverFiles
+module.exports = {
+  discoverFiles,
+  collectFilesInfo,
+}

packages/container/handleCommand.js

@@ -0,0 +1,152 @@
+const fs = require('fs').promises
+const minimist = require('minimist')
+const { discoverFiles, collectFilesInfo } = require('./discover')
+const { resolvePath } = require('./utils')
+
+const commandListeners = {
+  ls: {
+    opts: {
+      boolean: ['a', 'l', 'h'],
+    },
+    handler: discoverFiles,
+  },
+  cd: {
+    handler: changeDirectory,
+  },
+  touch: {
+    handler: createFiles,
+  },
+  mkdir: {
+    opts: {
+      boolean: ['p'],
+    },
+    handler: createFiles,
+  },
+  cp: {
+    opts: {
+      boolean: ['r'],
+    },
+    handler: copyFiles,
+  },
+  rm: {
+    opts: {
+      boolean: ['r', 'f', 'v'],
+    },
+    handler: removeFiles,
+  },
+  rmdir: {
+    opts: {
+      boolean: ['p', 'v'],
+    },
+    handler: removeFiles,
+  },
+  mv: {
+    handler: moveFiles,
+  },
+}
+
+async function handleCommand(command) {
+  const name = command.split(' ')[0]
+  const listener = commandListeners[name]
+  if (!listener) return {}
+  const argv = minimist(command.split(' '), listener.opts)
+  return listener.handler(argv)
+}
+
+function changeDirectory() {
+  return {
+    pwd: process.cwd(),
+  }
+}
+
+async function createFiles(argv) {
+  const { _ } = argv
+  const files = _.slice(1)
+  const results = []
+  for (const file of files) {
+    const filePath = resolvePath(file)
+    try {
+      const exists = await fs.stat(filePath)
+      if (exists) {
+        results.push({
+          path: filePath,
+          type: exists.isFile() ? 'file' : 'directory',
+          discovered: true,
+        })
+      }
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        throw err
+      }
+    }
+  }
+  return {
+    add: results,
+  }
+}
+
+async function copyFiles(argv) {
+  const { _ } = argv
+  const files = _.slice(1)
+  if (files.length === 0) {
+    return undefined
+  }
+  const dest = files.pop()
+  const destPath = resolvePath(dest)
+  return {
+    discover: await collectFilesInfo(destPath, 0, true),
+  }
+}
+
+async function removeFiles(argv) {
+  const files = argv._.slice(1)
+  const results = []
+  for (const file of files) {
+    const filePath = resolvePath(file)
+    try {
+      await fs.access(filePath, fs.constants.F_OK)
+    } catch {
+      results.push({
+        path: filePath,
+        type: 'file',
+        discovered: false,
+      })
+    }
+  }
+  return {
+    remove: results,
+  }
+}
+
+async function moveFiles(argv) {
+  const { _ } = argv
+  const sources = _.slice(1)
+  if (sources.length === 0) {
+    return undefined
+  }
+  const dest = sources.pop()
+  const results = {
+    add: [],
+    remove: [],
+  }
+  // check source
+  for (const source of sources) {
+    const sourcePath = resolvePath(source)
+    try {
+      await fs.access(sourcePath, fs.constants.F_OK)
+    } catch {
+      results.remove.push({
+        path: sourcePath,
+        type: 'file',
+        discovered: false,
+      })
+    }
+  }
+
+  // check dest
+  const destPath = resolvePath(dest)
+  results.add = await collectFilesInfo(destPath, 0, true)
+  return results
+}
+
+module.exports = handleCommand

packages/container/package.json

@@ -8,7 +8,6 @@
     "watch": ""
   },
   "dependencies": {
-    "axios": "^1.8.2",
     "minimist": "^1.2.8"
   }
 }

packages/container/utils.js

@@ -0,0 +1,10 @@
+const os = require('os')
+const path = require('path')
+
+function resolvePath(p) {
+  return path.resolve(process.cwd(), p.replace('~', os.homedir()))
+}
+
+module.exports = {
+  resolvePath,
+}

packages/file-graph/file.ts

@@ -4,6 +4,7 @@ export interface FileObject {
   path: string
   type: 'file' | 'directory'
   discovered: boolean
+  empty?: boolean
 }
 
 export default class File implements FileObject {
@@ -11,12 +12,14 @@ export default class File implements FileObject {
   name: string
   type: 'file' | 'directory'
   discovered: boolean
+  empty?: boolean
 
   constructor(obj: FileObject) {
     this.path = obj.path.trimEnd()
     this.name = basename(obj.path)
     this.type = obj.type
     this.discovered = obj.discovered
+    this.empty = obj.empty && obj.type === 'directory'
   }
 
   isDirectory() {

packages/file-graph/fileGraph.ts

@@ -45,17 +45,14 @@ export default class FileGraph extends FileNode {
 
   handleEvent(event: FileGraphUpdateEvent) {
     // eslint-disable-next-line default-case
-    switch (true) {
+    if (event.add) {
-      case !!event.add:
+      this.add(event.add)
-        this.add(event.add)
+    }
-        break
+    if (event.remove) {
-      case !!event.remove:
+      this.remove(event.remove)
-        this.remove(event.remove)
+    }
-        break
+    if (event.discover) {
-      case !!event.discover:
+      this.discover(event.discover)
-        this.discover(event.discover)
-        break
-      // Implement other cases if needed
     }
   }
 }

packages/file-graph/fileNode.ts

@@ -1,9 +1,5 @@
 import File, { FileObject } from './file.js'
-import {
+import { ParentNotExistsError, FileNotExistsError } from './errors.js'
-  DuplicateItemError,
-  ParentNotExistsError,
-  FileNotExistsError,
-} from './errors.js'
 import { dirname } from './utils.js'
 
 export interface IFileNode {
@@ -29,15 +25,24 @@ export default class FileNode extends File implements IFileNode {
   ) {
     const file = new FileNode(fileInput)
     if (file.path === this.path) {
-      throw new DuplicateItemError(
+      if (!this.discovered) {
-        `File ${file.path} already exists in the tree: ${this.toString()}`
+        this.discovered = file.discovered
-      )
+      }
+      return
     }
 
-    if (this.children.some((child) => child.path === file.path)) {
+    let existing = false
-      throw new DuplicateItemError(
+    for (const child of this.children) {
-        `File ${file.path} already exists in the tree: ${this.toString()}`
+      if (child.path === file.path) {
-      )
+        existing = true
+        if (!child.discovered) {
+          child.discovered = file.discovered
+        }
+        break
+      }
+    }
+    if (existing) {
+      return
     }
 
     const parentPath = dirname(file.path)
@@ -48,7 +53,7 @@ export default class FileNode extends File implements IFileNode {
       // Recursively call addChild on the correct child node
       const childNode = this.children.find((child) => child.contains(file))
       if (childNode) {
-        childNode.addChild(file)
+        childNode.addChild(file, { makeParents })
       } else if (makeParents) {
         const parent = new FileNode({
           path: parentPath,
@@ -95,6 +100,12 @@ export default class FileNode extends File implements IFileNode {
       // Update the state of this node
       this.discovered = this.discovered || fileNode.discovered
 
+      // the provided directory is empty, remove all children
+      if (fileNode.empty) {
+        this.children = []
+        return
+      }
+
       fileNode.children.forEach((child) => {
         const existingChild = this.children.find((c) => c.path === child.path)
         if (existingChild) {
@@ -140,4 +151,11 @@ export default class FileNode extends File implements IFileNode {
       .join('')
     return `${name}\n${childrenString}`
   }
+
+  include(filePath: string, type?: 'file' | 'directory'): boolean {
+    if (this.path === filePath) {
+      return type ? this.type === type : true
+    }
+    return this.children.some((child) => child.include(filePath, type))
+  }
 }

packages/file-graph/tests/fileGraph.test.ts

@@ -28,12 +28,44 @@ describe('FileGraph', () => {
         type: 'directory',
         discovered: true,
       },
+      {
+        path: '/home/user/folder2/file2.txt',
+        type: 'file',
+        discovered: true,
+      },
     ]
 
     fileGraph.add(filesToAdd)
 
     expect(fileGraph.children.length).toBe(1)
-    expect(fileGraph.children[0].children[0].children.length).toBe(2)
+    expect(fileGraph.children[0].children[0].children.length).toBe(3)
+  })
+
+  test('ignore files that already exist', () => {
+    const fileGraph = new FileGraph(root)
+
+    const filesToAdd: FileObject[] = [
+      {
+        path: '/home/user/folder1',
+        type: 'directory',
+        discovered: true,
+      },
+      {
+        path: '/home/user/folder1/file1.txt',
+        type: 'file',
+        discovered: true,
+      },
+    ]
+
+    fileGraph.add(filesToAdd)
+
+    expect(fileGraph.children.length).toBe(1)
+    expect(fileGraph.children[0].children[0].children.length).toBe(1)
+
+    fileGraph.add([filesToAdd[1]])
+
+    expect(fileGraph.children.length).toBe(1)
+    expect(fileGraph.children[0].children[0].children.length).toBe(1)
   })
 
   test('remove files from graph', () => {
@@ -175,4 +207,36 @@ describe('FileGraph', () => {
       3
     )
   })
+
+  test('discover a empty directory which children just being deleted', () => {
+    const fileGraph = new FileGraph(root)
+    const discoverFiles = [
+      {
+        path: '/home/user/folder1',
+        type: 'directory',
+        discovered: true,
+      },
+      {
+        path: '/home/user/folder1/file1.txt',
+        type: 'file',
+        discovered: true,
+      },
+    ]
+
+    fileGraph.add(discoverFiles as FileObject[])
+    expect(fileGraph.include('/home/user/folder1/file1.txt', 'file')).toBe(true)
+
+    fileGraph.discover([
+      {
+        path: '/home/user/folder1',
+        type: 'directory',
+        discovered: true,
+        empty: true,
+      },
+    ])
+
+    expect(fileGraph.include('/home/user/folder1/file1.txt', 'file')).toBe(
+      false
+    )
+  })
 })

packages/file-graph/tests/fileNode.test.ts

@@ -1,5 +1,5 @@
 import { describe, test, expect } from '@jest/globals'
-import { DuplicateItemError, ParentNotExistsError } from '../errors.js'
+import { ParentNotExistsError } from '../errors.js'
 import FileNode, { IFileNode } from '../fileNode.js'
 import { FileObject } from '../file.js'
 
@@ -47,11 +47,11 @@ describe('FileNode', () => {
     })
   })
 
-  test('should throw error when adding duplicate item', () => {
+  test('should override discovered when adding duplicate item', () => {
     const child: FileObject = {
       path: '/home/rudeus/test',
       type: 'directory',
-      discovered: true,
+      discovered: false,
     }
 
     // Adding the child for the first time
@@ -60,9 +60,10 @@ describe('FileNode', () => {
     expect(rootNode.children.length).toBe(1)
 
     // Trying to add the same child again should throw an error
-    expect(() => {
+    child.discovered = true
-      rootNode.addChild(child)
+    rootNode.addChild(child)
-    }).toThrow(DuplicateItemError)
+
+    expect(rootNode.children[0].discovered).toBe(true)
   })
 
   test('should throw error when parent does not exist', () => {

packages/game/schema/command.ts

@@ -4,10 +4,13 @@ const fileSchema = z.object({
   path: z.string(),
   type: z.enum(['file', 'directory']),
   discovered: z.boolean(),
+  empty: z.boolean().optional(),
 })
 
 export const commandParamsSchema = z.object({
   discover: fileSchema.array().optional(),
+  add: fileSchema.array().optional(),
+  remove: fileSchema.array().optional(),
   pwd: z.string().optional(),
 })
 

server/src/api/routes/commandRouter.ts

@@ -1,9 +1,8 @@
 import { Router } from 'express'
 import * as commandController from '../controllers/commandController.js'
-import { newCommandValidator } from '../validators/commandValidator.js'
 
 const router = Router()
 
-router.post('/', newCommandValidator, commandController.newCommand)
+router.post('/', commandController.newCommand)
 
 export default router

server/src/api/validators/commandValidator.ts

@@ -1,17 +0,0 @@
-import { body } from 'express-validator'
-import { noError } from '../../middleware/validator.js'
-
-const checkCommand = () => body('command').isString().notEmpty().trim()
-const checkPwd = () => body('pwd').isString().notEmpty().trim()
-const checkOutput = () => body('output').trim().optional()
-const checkError = () => body('error').trim().optional()
-const checkParams = () => body('params').isObject().optional()
-
-export const newCommandValidator = [
-  checkCommand(),
-  checkPwd(),
-  checkOutput(),
-  checkError(),
-  checkParams(),
-  noError,
-]

server/src/api/validators/questValidator.ts

@@ -1,7 +0,0 @@
-import { check } from 'express-validator'
-import { noError } from '../../middleware/validator.js'
-
-export const checkQuestId = () =>
-  check('questId').notEmpty().isString().isLowercase().isLength({ max: 64 })
-
-export const questIdValidator = [checkQuestId(), noError]

server/src/api/validators/sessionValidator.ts

@@ -1,14 +0,0 @@
-import { check } from 'express-validator'
-import { checkQuestId } from './questValidator.js'
-import { noError } from '../../middleware/validator.js'
-
-const checkSessionStatus = () =>
-  check('status').isString().isIn(['active', 'inactive', 'finished'])
-
-export const checkSessionId = () =>
-  check('sessionId').notEmpty().isString().isMongoId()
-
-export const sessionListValidator = [checkQuestId(), checkSessionStatus()]
-export const sessionDetailValidator = [checkSessionId()]
-export const createSessionValidator = [checkQuestId(), noError]
-export const activeSessionValidator = [checkQuestId(), noError]

server/src/config.ts

@@ -82,7 +82,8 @@ function createConfig() {
       network: get('DOCKER_NETWORK', 'linux-odyssey-players-dev'),
       defaultImage: get('QUEST_IMAGE', 'linuxodyssey/quest-base'),
       imagePrefix: get('DOCKER_PREFIX', 'linuxodyssey/quest-'),
-      mountQuest: process.env.MOUNT_QUEST === 'true',
+      mountQuest: !isProduction && process.env.MOUNT_QUEST === 'true',
+      mountCLI: !isProduction && process.env.MOUNT_CLI === 'true',
       hostProjectRoot: get('HOST_PROJECT_ROOT', projectRoot),
       keypairPath,
       keypair,

server/src/containers/docker.ts

@@ -39,13 +39,18 @@ export async function createContainer(
       `${config.docker.hostProjectRoot}/quests/${questId}/home:/etc/skel:ro`
     )
   }
-  if (!config.isProduction && config.docker.mountQuest && imageId !== 'base') {
+  if (config.docker.mountQuest && imageId !== 'base') {
     logger.info('Mounting quest directory', questId)
     binds = [
       `${config.docker.hostProjectRoot}/quests/${questId}/home:/home/commander`,
-      `${config.docker.hostProjectRoot}/packages/container:/usr/local/lib/container`,
     ]
   }
+  if (config.docker.mountCLI) {
+    logger.info('Mounting CLI', questId)
+    binds.push(
+      `${config.docker.hostProjectRoot}/packages/container:/usr/local/lib/container`
+    )
+  }
   const option = newContainerOptions(name, imageId, { binds })
   await createNetworkIfNotExists(config.docker.network)
   return engine.createContainer(option)

server/src/index.ts

@@ -38,6 +38,7 @@ async function main() {
   }
 
   if (config.testing.enabled) {
+    logger.info('Setting up test environment')
     await setupTest()
   }