api.file_list { search }
Massimo Melina committed
Dec 20, 2021 at 21:34 UTC
e19c701e266c12b9ba83a2d857eec7347ea0f9cc
6 files changed
+101
-46
dev-notes.md
+2
@@ -2,6 +2,8 @@
2
- search
3
try to use server-sent events for the reply
4
https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
5
+- file_list: option to decide what fields to get. If m && !c but we have no m, then copy from c.
6
+- api: gzip for >5kb
7
- frontend: dialogs
8
- vfs: serve an html for a folder?
9
- "default" property for vfsNode?
src/apis.ts
+22
-35
@@ -1,12 +1,10 @@
1
import Koa from 'koa'
2
-import { directPermOnNode, vfs, VfsNode } from './vfs'
3
-import { complySlashes, enforceFinal } from './misc'
2
+import { vfs, VfsNode, walkNode } from './vfs'
3
import { Stats } from 'fs'
4
import { stat } from 'fs/promises'
5
import _ from 'lodash'
6
import { getCurrentUser, verifyLogin } from './perm'
7
import { sessions } from './sessions'
9
-import glob from 'fast-glob'
8
9
export const SESSION_COOKIE = 'hfs_$id'
10
@@ -37,43 +35,32 @@ export function apiMw(apis: ApiHandlers) : Koa.Middleware {
35
}
36
}
37
38
+interface DirEntry { n:string, s?:number, m?:Date, c?:Date }
39
+
40
export const frontEndApis: ApiHandlers = {
41
- async file_list({ path, offset, limit }, ctx) {
41
+
42
+ async file_list({ path, offset, limit, search }, ctx) {
43
let node = await vfs.urlToNode(path || '/', ctx)
44
if (!node)
45
return
46
+ if (search?.includes('..'))
47
+ return ctx.throw(400)
48
+ const re = new RegExp(_.escapeRegExp(search),'i')
49
+ const match = (s?:string) => !s || !search || re.test(s)
50
const who = await getCurrentUser(ctx) // cache value
46
- const list = await Promise.all((node.children ||[]).map(node =>
47
- !node.hidden && directPermOnNode(node,who) && nodeToFile(node) ))
48
- _.remove(list, x => !x)
49
- if (offset)
50
- offset -= list.splice(0, offset).length
51
- const { source } = node
52
- if (list.length > limit)
53
- list.splice(limit)
54
- else if (source) {
55
- const base = enforceFinal('/', complySlashes(source)) // fast-glob lib wants forward-slashes
56
- const ignore = [node.hide, node.remove].flat().filter(Boolean).map(x => base!+x)
57
- const dirStream = glob.stream(base+'*', {
58
- dot: true,
59
- onlyFiles: false,
60
- ignore,
61
- })
62
- for await (let path of dirStream) {
63
- if (offset) {
64
- offset--
65
- continue
66
- }
67
- if (limit === list.length)
68
- break
69
- if (path instanceof Buffer)
70
- path = path.toString('utf8')
71
- const stats = await stat(path)
72
- const name = path.slice(base.length)
73
- list.push(statToFile(node!.rename?.[name] || name, stats))
51
+ const list = []
52
+ const walker = walkNode(node, who, search ? Infinity : 0)
53
+ for await (const sub of walker) {
54
+ if (!match(sub.name))
55
+ continue
56
+ if (offset) {
57
+ --offset
58
+ continue
59
}
60
+ list.push(await nodeToFile(sub))
61
+ if (limit === list.length)
62
+ break
63
}
76
-
64
return { list }
65
},
66
async login({ user, password }, ctx) {
@@ -112,9 +99,9 @@ export const frontEndApis: ApiHandlers = {
99
}
100
}
101
115
-async function nodeToFile(node: VfsNode) {
102
+async function nodeToFile(node: VfsNode): Promise<DirEntry | null> {
103
try {
117
- return node.source?.includes('//') ? { n:node.name }
104
+ return node.source?.includes('//') ? { n:node.name||'' }
105
: node.source ? statToFile(node.name, await stat(node.source))
106
: node.name ? { n: node.name + '/' }
107
: null
src/misc.ts
+1
-2
@@ -1,5 +1,4 @@
1
import fs from 'fs/promises'
2
-import glob from 'fast-glob'
2
import { objSameKeys } from './obj'
3
4
export function enforceFinal(sub:string, s:string) {
@@ -12,7 +11,7 @@ export async function isDirectory(path: string) {
11
}
12
13
export function complySlashes(path: string) {
15
- return glob.escapePath(path.replace(/\\/g,'/'))
14
+ return path.replace(/\\/g,'/')
15
}
16
17
export function prefix(pre:string, v:string|number, post:string='') {
src/vfs.ts
+53
-6
@@ -4,12 +4,15 @@ import fs from 'fs/promises'
4
import { FSWatcher, watch } from 'fs'
5
import { dirname, basename } from 'path'
6
import { isMatch } from 'micromatch'
7
-import { complySlashes, prefix, readFileBusy } from './misc'
7
+import { complySlashes, enforceFinal, prefix, readFileBusy } from './misc'
8
import { getCurrentUser } from './perm'
9
import Koa from 'koa'
10
+import glob from 'fast-glob'
11
+import _ from 'lodash'
12
11
-enum VfsNodeType {
13
+export enum VfsNodeType {
14
root,
15
+ temp,
16
}
17
18
export interface VfsNode {
@@ -75,7 +78,7 @@ export class Vfs {
78
}
79
}
80
78
- async urlToNode(url: string, ctx: Koa.Context) {
81
+ async urlToNode(url: string, ctx: Koa.Context) : Promise<VfsNode | undefined> {
82
const who = await getCurrentUser(ctx)
83
let run = this.root
84
const rest = url.split('/').filter(Boolean).map(decodeURIComponent)
@@ -89,12 +92,12 @@ export class Vfs {
92
continue
93
}
94
if (!run.source)
92
- return null
95
+ return
96
const relativeSource = piece + prefix('/', rest.join('/'))
94
- const baseSource = complySlashes(run.source+ '/') //TODO do we really need complySlashes here?
97
+ const baseSource = run.source+ '/'
98
const source = baseSource + relativeSource
99
const removed = isMatch(source, [run.remove].flat().map(x => baseSource + x))
97
- return removed || !await fs.stat(source) ? null : { source }
100
+ return removed || !await fs.stat(source) ? undefined : { source }
101
}
102
return run
103
@@ -120,3 +123,47 @@ export function directPermOnNode(node:VfsNode, username:string) {
123
const { perm } = node
124
return !perm ? 'r' : (username && perm[username] || perm['*'])
125
}
126
+
127
+
128
+export async function* walkNode(root:VfsNode, who:string, depth:number=0): AsyncIterableIterator<VfsNode> {
129
+ yield* recur(root, '', depth)
130
+
131
+ async function* recur(parent:VfsNode, prefixPath:string, depth:number): AsyncGenerator<VfsNode> {
132
+ const { children, source } = parent
133
+ if (children)
134
+ for (const c of children) {
135
+ if (c.hidden || !directPermOnNode(c,who))
136
+ continue
137
+ yield prefixPath ? { ...c, name: prefixPath+c.name } : c
138
+ if (depth > 0 && c)
139
+ yield* recur(c, prefixPath+c.name+'/', depth - 1)
140
+ }
141
+ if (!source)
142
+ return
143
+ const base = enforceFinal('/', complySlashes(source)) // fast-glob lib wants forward-slashes
144
+ const baseForGlob = glob.escapePath(base)
145
+ const ignore = [parent.hide, parent.remove].flat().filter(Boolean).map(x => baseForGlob+x)
146
+ const depthPath = depth === Infinity ? '**/' : _.repeat('*/',depth)
147
+ try {
148
+ const dirStream = glob.stream(baseForGlob + depthPath + '*', {
149
+ dot: true,
150
+ onlyFiles: false,
151
+ ignore,
152
+ })
153
+ for await (let path of dirStream) {
154
+ if (path instanceof Buffer)
155
+ path = path.toString('utf8')
156
+ const name = path.slice(base.length)
157
+ yield {
158
+ type: VfsNodeType.temp,
159
+ source: path,
160
+ name: parent!.rename?.[name] || name
161
+ }
162
+ }
163
+ }
164
+ catch(e) {
165
+ if ((e as any).code !== 'ENOTDIR')
166
+ throw e
167
+ }
168
+ }
169
+}
tests/test.ts
+22
-3
@@ -10,7 +10,15 @@ const appStarted = new Promise(resolve =>
10
describe('basics', () => {
11
//before(async () => appStarted)
12
it('frontend', req('/', s => s.includes('<body>')))
13
- it('api.list', req('/~/api/file_list', s => Array.isArray(s.list)))
13
+ it('api.list', req('/~/api/file_list', res => inList(res, 'f2/') && inList(res, 'f3/'), {
14
+ data: { path:'/f1/' }
15
+ }))
16
+ it('api.search', req('/~/api/file_list', res => inList(res, 'f2/') && !inList(res, 'f3/'), {
17
+ data: { path:'f1', search:'2' }
18
+ }))
19
+ it('api.search', req('/~/api/file_list', res => inList(res, 'f2/alfa.txt'), {
20
+ data: { path:'f1', search:'.txt' }
21
+ }))
22
it('download', req('/f1/f2/alfa.txt', s => s.includes('abcd')))
23
it('partial download', req('/f1/f2/alfa.txt', s => s.includes('a') && !s.includes('d'), {
24
headers: { Range: 'bytes=0-2' }
@@ -24,10 +32,17 @@ type Tester = number | ((data:any, fullResponse:any) => boolean | Error)
32
function req(methodUrl: string, test:Tester, requestOptions?:any) {
33
return (done:Done) => {
34
const i = methodUrl.indexOf('/')
27
- const method = methodUrl.slice(0,i) || 'GET'
35
+ const method = methodUrl.slice(0,i) || requestOptions?.data && 'POST' || 'GET'
36
const url = 'http://localhost'+methodUrl.slice(i)
37
function fun(res:any) {
30
- return done(typeof test === 'number' ? (res.status || res.response.status) !== test : !test(res.data, res))
38
+ if (typeof test === 'number') {
39
+ const ok = (res.status || res.response.status) === test
40
+ return done(!ok && 'expected code '+test)
41
+ }
42
+ const ok = test(res.data, res)
43
+ if (!ok)
44
+ console.debug('got',res.data)
45
+ done(!ok && Error())
46
}
47
axios.request({ method, url, ...requestOptions })
48
.then(fun, fun)
@@ -36,3 +51,7 @@ function req(methodUrl: string, test:Tester, requestOptions?:any) {
51
})
52
}
53
}
54
+
55
+function inList(res:any, name:string) {
56
+ return Array.isArray(res.list) && Boolean(res.list.find((x:any) => x.n===name))
57
+}
vfs.yaml
+1
@@ -4,6 +4,7 @@ children:
4
- name: f2
5
children:
6
- source: tests/alfa.txt
7
+ - name: f3
8
- name: proxy
9
source: https://raw.githubusercontent.com/nodejs/node/master/README.md
10
- name: for-rejetto