feat: file encoding support
mellbacon committed
Aug 12, 2023 at 22:18 UTC
ec3f956f66152eac60039d3ea8085e4aa4530cf2
6 files changed
+113
-22
src-tauri/src/encoding.rs
new
+29
@@ -0,0 +1,29 @@
1
+#[derive(PartialEq)]
2
+pub enum BOM {
3
+ BigEndian,
4
+ LittleEndian,
5
+ None
6
+}
7
+
8
+pub fn convert_to_u16(str: &str, bom: BOM) -> Vec<u8> {
9
+ let mut bytes = vec![];
10
+
11
+ if bom == BOM::BigEndian {
12
+ bytes = core::iter::once(0xFEFF)
13
+ .chain(str.encode_utf16())
14
+ .flat_map(|word| word.to_be_bytes())
15
+ .collect::<Vec<_>>();
16
+ }
17
+ else if bom == BOM::LittleEndian {
18
+ bytes = core::iter::once(0xFFFE)
19
+ .chain(str.encode_utf16())
20
+ .flat_map(|word| word.to_le_bytes())
21
+ .collect::<Vec<_>>();
22
+ }
23
+ else if bom == BOM::None {
24
+ bytes = str.encode_utf16()
25
+ .flat_map(|word| word.to_be_bytes())
26
+ .collect::<Vec<_>>();
27
+ }
28
+ bytes
29
+}
\ No newline at end of file
src-tauri/src/main.rs
+58
-5
@@ -3,11 +3,20 @@
3
windows_subsystem = "windows"
4
)]
5
6
+use std::ffi::OsStr;
7
+use std::fs::File;
8
+use std::io::Write;
9
+use std::path::Path;
10
use std::{env, fs};
11
use std::process::Command;
12
13
use tauri::Manager;
14
15
+mod encoding;
16
+use encoding::BOM;
17
+
18
+use crate::encoding::convert_to_u16;
19
+
20
#[tauri::command]
21
fn open_in_explorer(path: &str) {
22
// FOR OTHER OS REFER - https://doc.rust-lang.org/std/env/consts/constant.OS.html
@@ -51,17 +60,61 @@ fn delete_file(path: &str, perm: bool) {
60
}
61
}
62
63
+#[derive(Hash, Eq, PartialEq, Debug)]
64
+#[derive(serde::Serialize)]
65
+struct FileData {
66
+ text: String,
67
+ encoding: String,
68
+ extension: String,
69
+ bom: bool
70
+}
71
+
72
#[tauri::command]
55
-fn read_file(path: &str) -> std::string::String {
73
+fn read_file(path: &str) -> FileData {
74
let bytes = fs::read(path).unwrap();
57
- let content = encoding_rs::UTF_8.decode(&bytes);
58
- println!("{:?}", content);
59
- content.0.to_string()
75
+ let ext = Path::new(path).extension().and_then(OsStr::to_str).unwrap();
76
+ let file_data: FileData;
77
+
78
+ // encode based on bom if present otherwise just default to utf8
79
+ if let Some(data) = encoding_rs::Encoding::for_bom(&bytes) {
80
+ let (text, encoding, _) = data.0.decode(&bytes);
81
+ file_data = FileData {text: text.to_string(), encoding: encoding.name().to_string(), extension: ext.to_string(), bom: true}
82
+ }
83
+ else {
84
+ let (text, encoding, _) = encoding_rs::UTF_8.decode(&bytes);
85
+ file_data = FileData {text: text.to_string(), encoding: encoding.name().to_string(), extension: ext.to_string(), bom: false}
86
+ }
87
+ file_data
88
+}
89
+
90
+#[tauri::command]
91
+fn write_file(path: &str, content: &str, enc: &str, has_bom: bool) {
92
+ let mut output = Vec::new();
93
+ if let Some(data) = encoding_rs::Encoding::for_label(enc.as_bytes()) {
94
+ let (bytes, _, _) = data.encode(content);
95
+ let mut c_bytes = bytes.to_vec();
96
+ let mut bom: Vec<u8> = Vec::new();
97
+ if has_bom {
98
+ if enc == "UTF-8" {
99
+ bom = b"\xEF\xBB\xBF".to_vec();
100
+ }
101
+ else if enc == "UTF-16BE" {
102
+ c_bytes = convert_to_u16(content, BOM::BigEndian);
103
+ }
104
+ else if enc == "UTF-16LE" {
105
+ c_bytes = convert_to_u16(content, BOM::LittleEndian);
106
+ }
107
+ }
108
+ output = [bom.as_slice(), c_bytes.as_slice()].concat();
109
+ println!("{:?}", output.as_slice());
110
+ }
111
+ let mut file = File::create(path).unwrap();
112
+ file.write_all(&output).unwrap();
113
}
114
115
fn main() {
116
tauri::Builder::default()
64
- .invoke_handler(tauri::generate_handler![open_in_explorer, delete_file, attempt_file_access, is_file, is_folder, read_file])
117
+ .invoke_handler(tauri::generate_handler![open_in_explorer, delete_file, attempt_file_access, is_file, is_folder, read_file, write_file])
118
.plugin(tauri_plugin_fs_watch::init())
119
.plugin(tauri_plugin_store::Builder::default().build())
120
.run(tauri::generate_context!())
src/lib/Editor.svelte
+10
@@ -20,6 +20,8 @@
20
"path": "",
21
"fileType": "",
22
"language": "",
23
+ "encoding": "",
24
+ "hasBom": false,
25
"readonly": false,
26
});
27
@@ -35,6 +37,12 @@
37
export function getLang(ext) {
38
return getLangFromExt(ext);
39
}
40
+ export function getEncoding() {
41
+ return $file_info.encoding;
42
+ }
43
+ export function hasBom() {
44
+ return $file_info.hasBom;
45
+ }
46
export function getView() {
47
return editorView;
48
}
@@ -104,6 +112,7 @@
112
editorView.focus();
113
updateLineInfo();
114
language.set($file_info.language);
115
+ encoding.set({value: $file_info.encoding, hasBom: $file_info.hasBom});
116
}
117
export function updateLineInfo() {
118
let lineNumber = editorView.state.doc.lineAt(editorView.state.selection.main.head).number;
@@ -124,6 +133,7 @@
133
134
export const line_info = writable({line: "-", column: "-"});
135
export const language = writable("Unknown");
136
+ export const encoding = writable({value: "UTF-8", hasBom: false});
137
138
export function getLangFromExt(ext: string) {
139
if (ext === "txt") {
src/lib/File.ts
+5
-1
@@ -158,12 +158,16 @@ export async function saveFile(saveAs = false) {
158
}
159
// write changes to the file
160
// TODO: need to find a way to save files in different encodings. it saves as utf 8 by default (very annoying)
161
- fs.writeFile(tab.path, tab.content.getFileContent());
161
+ //fs.writeFile(tab.path, tab.content.getFileContent());
162
+ //const bytes = new TextEncoder().encode(tab.content.getFileContent())
163
+ await invoke("write_file", {path: tab.path, content: tab.content.getFileContent(), enc: tab.content.getEncoding(), hasBom: tab.content.hasBom()})
164
const fileType = await path.extname(tab.path);
165
tab.content.updateFileInfo({
166
"filename": tab.label,
167
"path": tab.path,
168
"fileType": fileType,
169
+ "encoding": tab.content.getEncoding(),
170
+ "hasBom": tab.content.hasBom(),
171
"language": tab.content.getLang(fileType),
172
"readonly": false,
173
});
src/lib/Statusbar.svelte
+2
-2
@@ -1,6 +1,6 @@
1
<script lang="ts">
2
import { isfile } from "./EditorTabList.svelte";
3
- import { line_info, language } from "./Editor.svelte";
3
+ import { line_info, language, encoding } from "./Editor.svelte";
4
</script>
5
6
<div id="statusbar">
@@ -15,7 +15,7 @@
15
<div class="editor-info">
16
<span title="Ln: {$line_info.line}, Col: {$line_info.column}">{$line_info.line} : {$line_info.column}</span>
17
<div class="divider"></div>
18
- <span>UTF-8</span>
18
+ <span>{$encoding.value} {$encoding.hasBom === true ? " with BOM" : ""}</span>
19
<div class="divider"></div>
20
<span>{$language}</span>
21
</div>
src/lib/Tab/Tab.ts
+9
-14
@@ -1,6 +1,6 @@
1
import { writable } from "svelte/store";
2
import Editor from "../Editor.svelte";
3
-import { path as p, dialog, invoke } from "@tauri-apps/api";
3
+import { dialog, invoke } from "@tauri-apps/api";
4
import { saveFile } from "../File";
5
6
export class Tab {
@@ -70,30 +70,25 @@ export class Tab {
70
if (this.tabOpen(path)) {
71
return;
72
}
73
- let fileContent = "";
73
+ let fileData = {text: "", encoding: "UTF-8", extension: "", bom: false};
74
try {
75
- fileContent = await invoke("read_file", {path: path}) //TODO: Fix performance issues/loading times on large files
76
- //console.log( await invoke("read_file", {path: path}));
75
+ // TODO: Fix performance issues/loading times on large files
76
+ fileData = await invoke("read_file", {path: path});
77
} catch (error) {
78
console.warn("Can't read file content. Setting to empty string. Error: ", error);
79
}
80
- let content = new Editor({target: document.getElementById("tabview"), props: {content: fileContent}});
80
+ let content = new Editor({target: document.getElementById("tabview"), props: {content: fileData.text}});
81
let tab = new this.Tab(this.id, label, content, path);
82
tab.isfile = true;
83
tab.saved = true;
84
- let fileType = "";
84
86
- try {
87
- fileType = await p.extname(tab.path);
88
- } catch (error) {
89
- console.warn("Cannot find file extension.")
90
- fileType = "";
91
- }
85
content.updateFileInfo({
86
"filename": tab.label,
87
"path": tab.path,
95
- "fileType": fileType,
96
- "language": content.getLang(fileType),
88
+ "fileType": fileData.extension,
89
+ "language": content.getLang(fileData.extension),
90
+ "encoding": fileData.encoding,
91
+ "hasBom": fileData.bom,
92
"readonly": false,
93
});
94
this.tablist = [...this.tablist, tab];