mrfrys-node-server/src/utils/files.ts

import { Schema } from 'jsonschema'
import {
    TestUsersSchema,
    isJsonValidAndLogError,
    PeersInfoSchema,
    ModulesSchema,
    SelfInfoSchema,
    LinksSchema,
    HttpsFilesSchema,
} from '../types/typeSchemas'
import logger from './logger'
import utils from './utils'
import { HttpsFiles } from '../types/basicTypes'

// FIXME: remove all file exists checks from everywhere for files that are created / checked here

type FileDescriptor = {
    path: string
    schema?: Schema
    defaultValue?: string
    description?: string
    errorIfMissing?: boolean
    warningIfMissing?: boolean
}

export const validateAndSetupFiles = (): boolean => {
    let everythingValid = true
    Object.entries(files).forEach(([key, file]: [string, FileDescriptor]) => {
        let fileExists = utils.FileExists(file.path)
        if (file.defaultValue != null && !fileExists) {
            // FIXME: create path too
            utils.WriteFile(file.defaultValue, file.path)
            fileExists = true
        }

        if (file.errorIfMissing && !fileExists) {
            const errMsg = [`File "${file.path}" does not exist! (${key})`]
            if (file.description) {
                errMsg.push(`Should be: ${file.description}`)
            }
            logger.Log(errMsg.join(' '), 'redbg')
            everythingValid = false
            return
        }

        if (file.warningIfMissing && !fileExists) {
            const warningMsg = [
                `Optional file "${file.path}" does not exist! (${key})`,
            ]
            if (file.description) {
                warningMsg.push(`Should be: ${file.description}`)
            }
            logger.Log(warningMsg.join(' '), 'yellowbg')
        }

        if (file.schema && fileExists) {
            const val = utils.ReadJSON(file.path)
            if (!isJsonValidAndLogError(val, file.schema, file.path)) {
                everythingValid = false
                return
            }
        }
    })

    return everythingValid
}

export const readAndValidateFile = <T>(file: FileDescriptor): T => {
    if (!file.schema) return null
    const fileExists = utils.FileExists(file.path)
    if (!fileExists) return null

    const content = utils.ReadFile(file.path)
    const parsedContent: T = JSON.parse(content)

    if (!isJsonValidAndLogError(parsedContent, file.schema, file.path))
        return null

    return parsedContent
}

const filesFiles = {
    httpsFiles: {
        path: 'data/httpsfiles.json',
        description:
            'file paths for https functionality (privkey, chain files)',
        schema: HttpsFilesSchema,
        warningIfMissing: true,
    },
} as const satisfies Record<string, FileDescriptor>

const httpsFiles: HttpsFiles = utils.FileExists(filesFiles.httpsFiles.path)
    ? utils.ReadJSON<HttpsFiles>(filesFiles.httpsFiles.path)
    : null

export const files = {
    ...filesFiles,
    // --------------------------------------------------------------------------------
    // server / modules files
    // --------------------------------------------------------------------------------
    serverPath: {
        path: 'dist/server.js',
        errorIfMissing: true,
        description:
            'server main entry file, created after running "npm run build"',
    },
    qminingPageDir: {
        path: 'submodules/qmining-page',
        errorIfMissing: true,
        description:
            'qmining page submodule directory, created by pulling submodules / setup script',
    },
    qminingIndexPath: {
        path: 'nextStatic/qminingPagePublic/index.html',
        errorIfMissing: true,
        description:
            'qmining page-s build index.html, created by "npm run build" in qmining page submodule dir',
    },
    dataEditorPageDir: {
        path: 'submodules/qmining-data-editor',
        errorIfMissing: true,
        description:
            'qmining data editor page submodule directory, created by pulling submodules / setup script',
    },
    dataEditorIndexPath: {
        path: 'nextStatic/dataEditorPublic/index.html',
        errorIfMissing: true,
        description:
            'qmining data editor-s build index.html, created by "npm run build" in qmining data editor submodule dir',
    },
    moodleTestUserscriptDir: {
        path: 'submodules/moodle-test-userscript',
        errorIfMissing: true,
        description:
            'moodle test userscript submodule directory, created by pulling submodules / setup script',
    },
    moodleTestUserscriptPath: {
        path: 'submodules/moodle-test-userscript/stable.user.js',
        errorIfMissing: true,
        description:
            'moodle test userscript file, created by pulling submodules / setup script',
    },

    domainFile: {
        path: 'data/domain',
        description:
            'server domain for cookies and stuff, for ex.: "qmining.com", no "http://" and things like that, just the domain',
    },
    // --------------------------------------------------------------------------------
    // stats files
    // --------------------------------------------------------------------------------
    registeredScriptsFile: {
        path: 'stats/registeredScripts.json',
        defaultValue: JSON.stringify([]),
    },
    askedQuestionFile: {
        path: 'stats/askedQuestions',
        description: 'text file of recieved data on /ask',
    },
    recievedQuestionFile: {
        path: 'stats/recievedQuestions',
        description: 'text file of recieved data on /isAdding',
    },
    dailyDataCountFile: {
        path: 'stats/dailyDataCount',
        description: 'text file of daily data count',
    },
    dataEditsLog: {
        path: 'stats/dataEdits',
        description: 'text file of data edit logs',
    },

    // --------------------------------------------------------------------------------
    // https files
    // --------------------------------------------------------------------------------
    privkeyFile: {
        path: httpsFiles?.privkeyFile,
        description: 'private key file for https',
    },
    fullchainFile: {
        path: httpsFiles?.fullchainFile,
        description: 'full chain key file for https',
    },
    chainFile: {
        path: httpsFiles?.chainFile,
        description: 'chain key file for https',
    },

    // --------------------------------------------------------------------------------
    // api files
    // --------------------------------------------------------------------------------
    rootRedirectToFile: {
        path: 'data/apiRootRedirectTo',
        description: 'url to redirect users trying to acces root api path',
        warningIfMissing: true,
    },
    modulesFile: {
        path: './src/modules.json',
        errorIfMissing: true,
        description: 'module files for server',
        schema: ModulesSchema,
    },
    extraModulesFile: {
        path: './src/extraModules/extraModules.json',
        description: 'extra private modules for server, not tracked by git',
        schema: ModulesSchema,
    },
    statExcludeFile: {
        path: './data/statExclude.json',
        errorIfMissing: true,
        description:
            'array of strings which if included in requests url-s then the request itself is not counted in stats',
        defaultValue: JSON.stringify([]),
        schema: { type: 'array', items: { type: 'string' } },
    },
    usersDBPath: {
        path: './data/dbs/users.db',
        errorIfMissing: true,
        description: 'users sqlite db file',
    },

    // --------------------------------------------------------------------------------
    // qmining api
    // --------------------------------------------------------------------------------
    testUsersFile: {
        path: 'data/testUsers.json',
        defaultValue: JSON.stringify({ userIds: [] }),
        schema: TestUsersSchema,
        description:
            'test users, recieved data from them wont get added to question dbs',
    },
    linksFile: {
        path: 'data/links.json',
        schema: LinksSchema,
        description: 'file containing links, for ex.: patreon and paypal',
        warningIfMissing: true,
    },

    // --------------------------------------------------------------------------------
    // log files
    // --------------------------------------------------------------------------------
    vlogDir: {
        path: 'stats/vlogs/',
        description: 'verbose logs directory',
    },
    logDir: {
        path: 'stats/logs/',
        description: 'basic logs directory',
    },
    statFile: {
        path: 'stats/stats',
        defaultValue: JSON.stringify({}),
        description: 'json of visited paths all time',
    },
    vStatFile: {
        path: 'stats/vstats',
        defaultValue: JSON.stringify({}),
        description: 'json of visited paths by day',
    },
    uStatsFile: {
        path: 'stats/ustats',
        defaultValue: JSON.stringify({}),
        description: 'json of visits per user',
    },
    uvStatsFile: {
        path: 'stats/uvstats',
        defaultValue: JSON.stringify({}),
        description: 'json of visits per user by day',
    },
    idstatsFile: {
        path: './stats/idstats',
        defaultValue: JSON.stringify({}),
        description: 'json of various statistics per user id',
    },
    idvstatsFile: {
        path: './stats/idvstats',
        defaultValue: JSON.stringify({}),
        description: 'json of various statistics per user id per day',
    },
    nologFile: {
        path: './data/nolog',
        defaultValue: '',
        description:
            'text file of users seperated by new lines to ignore in logging / stats',
    },

    // --------------------------------------------------------------------------------
    // peer files
    // --------------------------------------------------------------------------------
    peersPath: {
        path: 'data/p2p/',
        description: 'p2p files directory',
    },
    peersFile: {
        path: 'data/p2p/peers.json',
        description: 'json of list of peers',
        defaultValue: JSON.stringify([]),
        schema: PeersInfoSchema,
    },
    selfInfoFile: {
        path: 'data/p2p/selfInfo.json',
        description: 'json of info of this servers peer functionality',
        defaultValue: JSON.stringify({}),
        schema: SelfInfoSchema,
    },
    thirdPartyPeersFile: {
        path: 'data/p2p/thirdPartyPeers.json',
        defaultValue: JSON.stringify([]),
        description: 'json of third party peers reported by other peers',
    },
    keyFile: {
        path: 'data/p2p/key',
    },
} as const satisfies Record<string, FileDescriptor>

export const paths = Object.entries(files).reduce(
    (acc, [key, value]) => ({ ...acc, [key]: value.path }),
    {}
) as Record<keyof typeof files, string>