frontend: use SSE for search
Massimo Melina committed
Dec 22, 2021 at 14:42 UTC
bdb1d9002c2e700375f24c8ef17d15834f46edeb
6 files changed
+122
-44
frontend/package.json
+1
-1
@@ -1,6 +1,6 @@
1
{
2
"name": "frontend",
3
- "version": "0.1.0",
3
+ "version": "0.3.0",
4
"private": true,
5
"proxy": "http://localhost",
6
"scripts": {
frontend/src/BrowseFiles.ts
+88
-34
@@ -1,6 +1,6 @@
1
import { Link, useLocation } from 'react-router-dom'
2
-import { useApi } from './api'
3
-import { createContext, createElement as h, Fragment, useContext, useEffect, useMemo, useState } from 'react'
2
+import { apiCall, apiEvents } from './api'
3
+import { createContext, createElement as h, Fragment, useContext, useEffect, useRef, useState } from 'react'
4
import { formatBytes, hError, hIcon } from './misc'
5
import { Spinner } from './components'
6
import { Head } from './Head'
@@ -11,58 +11,113 @@ function usePath() {
11
return decodeURI(useLocation().pathname)
12
}
13
14
-export const ListContext = createContext<{ list:DirList, unfinished:boolean }>({ list:[], unfinished: false })
14
+interface DirEntry { n:string, s?:number, m?:string, c?:string }
15
+export type DirList = DirEntry[]
16
+interface ListRes { list:DirList, unfinished?:boolean, err?:Error }
17
+
18
+export const ListContext = createContext<ListRes>({ list:[], unfinished: false })
19
20
export function BrowseFiles() {
17
- const [list, unfinished] = useFetchList()
21
+ const { list, unfinished, error } = useFetchList()
22
+ if (error)
23
+ return hError(error)
24
if (!list)
25
return h(Spinner)
20
- if (list instanceof Error)
21
- return hError(list)
26
return h(ListContext.Provider, { value:{ list, unfinished } },
27
h(Head),
28
h(FilesList))
29
}
30
31
function useFetchList() {
32
+ const snap = useSnapState()
33
const desiredPath = usePath()
29
- const PRELOAD_SIZE = 100
30
- const [preloading, setPreloading] = useState(true)
31
- const [path, setPath] = useState('')
34
+ const search = snap.remoteSearch || undefined
35
+ const [list, setList] = useState<DirList>([])
36
+ const [unfinished, setUnfinished] = useState(true)
37
+ const [error, setError] = useState<Error>()
38
+ const lastPath = useRef('')
39
useEffect(()=>{
40
const loc = window.location
34
- if (!desiredPath.endsWith('/')) // useful only in dev, while accessing the frontend directly without passing by the main server
35
- loc.href = loc.href+'/'
36
- state.remoteSearch = ''
37
- setPreloading(true)
38
- setPath(desiredPath)
39
- }, [desiredPath])
40
- const API = 'file_list'
41
- const snap = useSnapState()
42
- const search = snap.remoteSearch
43
- const baseParams = { path, search, omit:'c' }
44
- const preload = useApi(path && API, { ...baseParams, limit: PRELOAD_SIZE })
45
- const rest = useApi(!preloading && API, { ...baseParams, offset: PRELOAD_SIZE })
46
- const list = useMemo(() => !preload ? null
47
- : !rest ? (preload.list || preload) // the || is for an Error instance
48
- : [...preload.list, ...rest.list],
49
- [preload, rest])
50
- const unfinished = preload && !rest && list.length === PRELOAD_SIZE
51
- if (unfinished && preloading) // let load it all
52
- setPreloading(false)
53
- return [ list, unfinished ]
54
-}
41
+ if (!desiredPath.endsWith('/')) { // useful only in dev, while accessing the frontend directly without passing by the main server
42
+ loc.href = loc.href + '/'
43
+ return
44
+ }
45
+ const previous = lastPath.current
46
+ lastPath.current = desiredPath
47
+ if (previous !== desiredPath && search) {
48
+ state.remoteSearch = ''
49
+ state.stopSearch?.()
50
+ return
51
+ }
52
56
-interface DirEntry { n:string, s?:number, m?:string, c?:string }
57
-export type DirList = DirEntry[]
53
+ ;(async ()=>{
54
+ const API = 'file_list'
55
+ const sse = search
56
+ const baseParams = { path:desiredPath, search, sse, omit:'c' }
57
+ let list: DirList = []
58
+ setUnfinished(true)
59
+ setList(list)
60
+
61
+ if (sse) { // buffering entries is necessary against burst of events that will hang the browser
62
+ const buffer:DirList = []
63
+ const flush = () => {
64
+ const chunk = buffer.splice(0, Infinity)
65
+ if (chunk.length)
66
+ setList(list = [...list, ...chunk])
67
+ }
68
+ const timer = setInterval(flush, 1000)
69
+ const src = apiEvents(API, baseParams, (type, data) => {
70
+ switch (type) {
71
+ case 'error':
72
+ return setError(Error(JSON.stringify(data)))
73
+ case 'closed':
74
+ clearInterval(timer)
75
+ flush()
76
+ return setUnfinished(false)
77
+ case 'msg':
78
+ if (src?.readyState === src?.CLOSED)
79
+ return state.stopSearch?.()
80
+ let { entry } = data
81
+ console.log(entry.n)
82
+ buffer.push(entry)
83
+ }
84
+ })
85
+ state.stopSearch = ()=>{
86
+ buffer.length = 0
87
+ clearInterval(timer)
88
+ state.stopSearch = undefined
89
+ src.close()
90
+ }
91
+ return
92
+ }
93
+
94
+ let offset = 0
95
+ while (1) {
96
+ const limit = list.length ? 1000 : 100
97
+ const res = await apiCall(API, { ...baseParams, offset, limit })
98
+ || Error()
99
+ if (res instanceof Error)
100
+ return setError(res)
101
+ const chunk = res.list
102
+ setList(list = [ ...list, ...chunk ])
103
+ if (chunk.length < limit)
104
+ break
105
+ offset = list.length
106
+ }
107
+ setUnfinished(false)
108
+ })()
109
+ }, [desiredPath, search])
110
+ return { list, unfinished, error }
111
+}
112
113
function FilesList() {
114
const { list, unfinished } = useContext(ListContext)
115
const snap = useSnapState()
116
+ if (!list) return null
117
const filter = snap.listFilter > '' && new RegExp(_.escapeRegExp(snap.listFilter),'i')
118
let n = 0 // if I try to use directly the state as counter I get a "too many re-renders" error
119
const ret = h('ul', { className: 'dir' },
65
- !list.length ? 'Nothing here'
120
+ !list.length ? (unfinished || 'Nothing here')
121
: list.map((entry: DirEntry) =>
122
h(File, { key: entry.n, hidden: filter && !filter.test(entry.n) || !++n, ...entry })),
123
unfinished && h(Spinner))
@@ -88,4 +143,3 @@ function File({ n, m, c, s, hidden }: DirEntry & { hidden:boolean }) {
143
h('div', { style:{ clear:'both' } })
144
)
145
}
91
-
frontend/src/api.ts
+24
-1
@@ -1,8 +1,10 @@
1
import { useEffect, useState } from 'react';
2
import { Falsy } from './misc'
3
4
+const PREFIX = '/~/api/'
5
+
6
export function apiCall(cmd: string, params?: object) : Promise<any> {
5
- return fetch('/~/api/'+cmd, {
7
+ return fetch(PREFIX+cmd, {
8
method: 'POST',
9
headers: { 'content-type': 'application/json' },
10
body: params && JSON.stringify(params),
@@ -30,3 +32,24 @@ export function useApi(cmd: string | Falsy, params?: object) : any {
32
}, [cmd, JSON.stringify(params)]) //eslint-disable-line
33
return x
34
}
35
+
36
+type EventHandler = (type:string, data?:any) => void
37
+
38
+export function apiEvents(cmd: string, params: object, cb:EventHandler) {
39
+ const source = new EventSource(PREFIX + cmd + '?' + new URLSearchParams(params as any))
40
+ source.onopen = () => cb('connected')
41
+ source.onerror = err => cb('error', err)
42
+ source.onmessage = ({ data }) => {
43
+ if (!data) {
44
+ cb('closed')
45
+ return source.close()
46
+ }
47
+ try { data = JSON.parse(data) }
48
+ catch(e) {
49
+ return cb('string', data)
50
+ }
51
+ cb('msg', data)
52
+ }
53
+ return source
54
+}
55
+
frontend/src/state.ts
+8
-1
@@ -1,6 +1,13 @@
1
import { proxy, useSnapshot } from 'valtio'
2
3
-export const state = proxy({
3
+export const state = proxy<{
4
+ stopSearch?: ()=>void,
5
+ iconsClass: string,
6
+ username: string,
7
+ listFilter: string,
8
+ remoteSearch: string,
9
+ filteredEntries: number,
10
+}>({
11
iconsClass: '',
12
username: '',
13
listFilter: '',
tests/test.ts
+1
-4
@@ -16,9 +16,6 @@ describe('basics', () => {
16
it('api.search', req('/~/api/file_list', res => inList(res, 'f2/') && !inList(res, 'f3/'), {
17
data: { path:'f1', search:'2' }
18
}))
19
- it('api.search', req('/~/api/file_list', res => inList(res, 'f2/alfa.txt'), {
20
- data: { path:'f1', search:'.txt' }
21
- }))
19
it('download', req('/f1/f2/alfa.txt', s => s.includes('abcd')))
20
it('partial download', req('/f1/f2/alfa.txt', s => s.includes('a') && !s.includes('d'), {
21
headers: { Range: 'bytes=0-2' }
@@ -41,7 +38,7 @@ function req(methodUrl: string, test:Tester, requestOptions?:any) {
38
}
39
const ok = test(res.data, res)
40
if (!ok)
44
- console.debug('got',res.data)
41
+ console.debug('sent', requestOptions, 'got',res.data)
42
done(!ok && Error())
43
}
44
axios.request({ method, url, ...requestOptions })
todo.md
-3
@@ -1,8 +1,5 @@
1
# To do
2
-- search: try to use server-sent events for the reply
3
- https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
2
- node.link
5
-- frontend cache should update on file change
3
- vfs: serve an html for a folder?
4
- "default" property for vfsNode?
5
- interruption of long requests if client aborted (searching/listing)