@cryptotaxi247 / netdata-1 / commits / 0ba3827c5

Add better recovery for corrupted metadata (#15891)

* Add sqlite-meta-recover command line option Remove the old recovery that would attempt to fix only chart and dimension Mark recovery for metadata (for now) Simplify the database init function * Reduce variable scope, formatting

Stelios Fragkakis committed Sep 1, 2023 at 17:35 UTC 0ba3827c53b753c493fed561be3a700e61751fa9
10 files changed +4206 -191
CMakeLists.txt
+3
@@ -771,6 +771,9 @@ set(RRD_PLUGIN_FILES
771 database/sqlite/sqlite_aclk_alert.h
772 database/sqlite/sqlite3.c
773 database/sqlite/sqlite3.h
774 + database/sqlite/sqlite3recover.c
775 + database/sqlite/sqlite3recover.h
776 + database/sqlite/dbdata.c
777 database/engine/rrdengine.c
778 database/engine/rrdengine.h
779 database/engine/rrddiskprotocol.h
Makefile.am
+3
@@ -505,6 +505,9 @@ RRD_PLUGIN_FILES = \
505 database/sqlite/sqlite_aclk_alert.h \
506 database/sqlite/sqlite3.c \
507 database/sqlite/sqlite3.h \
508 + database/sqlite/sqlite3recover.c \
509 + database/sqlite/sqlite3recover.h \
510 + database/sqlite/dbdata.c \
511 database/KolmogorovSmirnovDist.c \
512 database/KolmogorovSmirnovDist.h \
513 $(NULL)
daemon/main.c
+4 -9
@@ -787,8 +787,7 @@ int help(int exitcode) {
787 " -W stacksize=N Set the stacksize (in bytes).\n\n"
788 " -W debug_flags=N Set runtime tracing to debug.log.\n\n"
789 " -W unittest Run internal unittests and exit.\n\n"
790 - " -W sqlite-check Check metadata database integrity and exit.\n\n"
791 - " -W sqlite-fix Check metadata database integrity, fix if needed and exit.\n\n"
790 + " -W sqlite-meta-recover Run recovery on the metadata database and exit.\n\n"
791 " -W sqlite-compact Reclaim metadata database unused space and exit.\n\n"
792 #ifdef ENABLE_DBENGINE
793 " -W createdataset=N Create a DB engine dataset of N seconds and exit.\n\n"
@@ -1436,13 +1435,9 @@ int main(int argc, char **argv) {
1435 char* createdataset_string = "createdataset=";
1436 char* stresstest_string = "stresstest=";
1437 #endif
1439 - if(strcmp(optarg, "sqlite-check") == 0) {
1440 - sql_init_database(DB_CHECK_INTEGRITY, 0);
1441 - return 0;
1442 - }
1438
1444 - if(strcmp(optarg, "sqlite-fix") == 0) {
1445 - sql_init_database(DB_CHECK_FIX_DB, 0);
1439 + if(strcmp(optarg, "sqlite-meta-recover") == 0) {
1440 + sql_init_database(DB_CHECK_RECOVER, 0);
1441 return 0;
1442 }
1443
@@ -1509,7 +1504,7 @@ int main(int argc, char **argv) {
1504 unittest_running = true;
1505 return aral_unittest(10000);
1506 }
1512 - else if(strcmp(optarg, "stringtest") == 0) {
1507 + else if(strcmp(optarg, "stringtest") == 0) {
1508 unittest_running = true;
1509 return string_unittest(10000);
1510 }
database/sqlite/dbdata.c new
+959
@@ -0,0 +1,959 @@
1 +/*
2 +** 2019-04-17
3 +**
4 +** The author disclaims copyright to this source code. In place of
5 +** a legal notice, here is a blessing:
6 +**
7 +** May you do good and not evil.
8 +** May you find forgiveness for yourself and forgive others.
9 +** May you share freely, never taking more than you give.
10 +**
11 +******************************************************************************
12 +**
13 +** This file contains an implementation of two eponymous virtual tables,
14 +** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the
15 +** "sqlite_dbpage" eponymous virtual table be available.
16 +**
17 +** SQLITE_DBDATA:
18 +** sqlite_dbdata is used to extract data directly from a database b-tree
19 +** page and its associated overflow pages, bypassing the b-tree layer.
20 +** The table schema is equivalent to:
21 +**
22 +** CREATE TABLE sqlite_dbdata(
23 +** pgno INTEGER,
24 +** cell INTEGER,
25 +** field INTEGER,
26 +** value ANY,
27 +** schema TEXT HIDDEN
28 +** );
29 +**
30 +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE
31 +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND
32 +** "schema".
33 +**
34 +** Each page of the database is inspected. If it cannot be interpreted as
35 +** a b-tree page, or if it is a b-tree page containing 0 entries, the
36 +** sqlite_dbdata table contains no rows for that page. Otherwise, the
37 +** table contains one row for each field in the record associated with
38 +** each cell on the page. For intkey b-trees, the key value is stored in
39 +** field -1.
40 +**
41 +** For example, for the database:
42 +**
43 +** CREATE TABLE t1(a, b); -- root page is page 2
44 +** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five');
45 +** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten');
46 +**
47 +** the sqlite_dbdata table contains, as well as from entries related to
48 +** page 1, content equivalent to:
49 +**
50 +** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES
51 +** (2, 0, -1, 5 ),
52 +** (2, 0, 0, 'v' ),
53 +** (2, 0, 1, 'five'),
54 +** (2, 1, -1, 10 ),
55 +** (2, 1, 0, 'x' ),
56 +** (2, 1, 1, 'ten' );
57 +**
58 +** If database corruption is encountered, this module does not report an
59 +** error. Instead, it attempts to extract as much data as possible and
60 +** ignores the corruption.
61 +**
62 +** SQLITE_DBPTR:
63 +** The sqlite_dbptr table has the following schema:
64 +**
65 +** CREATE TABLE sqlite_dbptr(
66 +** pgno INTEGER,
67 +** child INTEGER,
68 +** schema TEXT HIDDEN
69 +** );
70 +**
71 +** It contains one entry for each b-tree pointer between a parent and
72 +** child page in the database.
73 +*/
74 +
75 +#pragma GCC diagnostic push
76 +#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
77 +#pragma GCC diagnostic ignored "-Wunused-parameter"
78 +#if !defined(SQLITEINT_H)
79 +#include "sqlite3.h"
80 +
81 +typedef unsigned char u8;
82 +typedef unsigned int u32;
83 +
84 +#endif
85 +#include <string.h>
86 +#include <assert.h>
87 +
88 +#ifndef SQLITE_OMIT_VIRTUALTABLE
89 +
90 +#define DBDATA_PADDING_BYTES 100
91 +
92 +typedef struct DbdataTable DbdataTable;
93 +typedef struct DbdataCursor DbdataCursor;
94 +
95 +/* Cursor object */
96 +struct DbdataCursor {
97 + sqlite3_vtab_cursor base; /* Base class. Must be first */
98 + sqlite3_stmt *pStmt; /* For fetching database pages */
99 +
100 + int iPgno; /* Current page number */
101 + u8 *aPage; /* Buffer containing page */
102 + int nPage; /* Size of aPage[] in bytes */
103 + int nCell; /* Number of cells on aPage[] */
104 + int iCell; /* Current cell number */
105 + int bOnePage; /* True to stop after one page */
106 + int szDb;
107 + sqlite3_int64 iRowid;
108 +
109 + /* Only for the sqlite_dbdata table */
110 + u8 *pRec; /* Buffer containing current record */
111 + sqlite3_int64 nRec; /* Size of pRec[] in bytes */
112 + sqlite3_int64 nHdr; /* Size of header in bytes */
113 + int iField; /* Current field number */
114 + u8 *pHdrPtr;
115 + u8 *pPtr;
116 + u32 enc; /* Text encoding */
117 +
118 + sqlite3_int64 iIntkey; /* Integer key value */
119 +};
120 +
121 +/* Table object */
122 +struct DbdataTable {
123 + sqlite3_vtab base; /* Base class. Must be first */
124 + sqlite3 *db; /* The database connection */
125 + sqlite3_stmt *pStmt; /* For fetching database pages */
126 + int bPtr; /* True for sqlite3_dbptr table */
127 +};
128 +
129 +/* Column and schema definitions for sqlite_dbdata */
130 +#define DBDATA_COLUMN_PGNO 0
131 +#define DBDATA_COLUMN_CELL 1
132 +#define DBDATA_COLUMN_FIELD 2
133 +#define DBDATA_COLUMN_VALUE 3
134 +#define DBDATA_COLUMN_SCHEMA 4
135 +#define DBDATA_SCHEMA \
136 + "CREATE TABLE x(" \
137 + " pgno INTEGER," \
138 + " cell INTEGER," \
139 + " field INTEGER," \
140 + " value ANY," \
141 + " schema TEXT HIDDEN" \
142 + ")"
143 +
144 +/* Column and schema definitions for sqlite_dbptr */
145 +#define DBPTR_COLUMN_PGNO 0
146 +#define DBPTR_COLUMN_CHILD 1
147 +#define DBPTR_COLUMN_SCHEMA 2
148 +#define DBPTR_SCHEMA \
149 + "CREATE TABLE x(" \
150 + " pgno INTEGER," \
151 + " child INTEGER," \
152 + " schema TEXT HIDDEN" \
153 + ")"
154 +
155 +/*
156 +** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual
157 +** table.
158 +*/
159 +static int dbdataConnect(
160 + sqlite3 *db,
161 + void *pAux,
162 + int argc, const char *const*argv,
163 + sqlite3_vtab **ppVtab,
164 + char **pzErr
165 +){
166 + DbdataTable *pTab = 0;
167 + int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA);
168 +
169 + (void)argc;
170 + (void)argv;
171 + (void)pzErr;
172 + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS);
173 + if( rc==SQLITE_OK ){
174 + pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable));
175 + if( pTab==0 ){
176 + rc = SQLITE_NOMEM;
177 + }else{
178 + memset(pTab, 0, sizeof(DbdataTable));
179 + pTab->db = db;
180 + pTab->bPtr = (pAux!=0);
181 + }
182 + }
183 +
184 + *ppVtab = (sqlite3_vtab*)pTab;
185 + return rc;
186 +}
187 +
188 +/*
189 +** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table.
190 +*/
191 +static int dbdataDisconnect(sqlite3_vtab *pVtab){
192 + DbdataTable *pTab = (DbdataTable*)pVtab;
193 + if( pTab ){
194 + sqlite3_finalize(pTab->pStmt);
195 + sqlite3_free(pVtab);
196 + }
197 + return SQLITE_OK;
198 +}
199 +
200 +/*
201 +** This function interprets two types of constraints:
202 +**
203 +** schema=?
204 +** pgno=?
205 +**
206 +** If neither are present, idxNum is set to 0. If schema=? is present,
207 +** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit
208 +** in idxNum is set.
209 +**
210 +** If both parameters are present, schema is in position 0 and pgno in
211 +** position 1.
212 +*/
213 +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){
214 + DbdataTable *pTab = (DbdataTable*)tab;
215 + int i;
216 + int iSchema = -1;
217 + int iPgno = -1;
218 + int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA);
219 +
220 + for(i=0; i<pIdx->nConstraint; i++){
221 + struct sqlite3_index_constraint *p = &pIdx->aConstraint[i];
222 + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
223 + if( p->iColumn==colSchema ){
224 + if( p->usable==0 ) return SQLITE_CONSTRAINT;
225 + iSchema = i;
226 + }
227 + if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){
228 + iPgno = i;
229 + }
230 + }
231 + }
232 +
233 + if( iSchema>=0 ){
234 + pIdx->aConstraintUsage[iSchema].argvIndex = 1;
235 + pIdx->aConstraintUsage[iSchema].omit = 1;
236 + }
237 + if( iPgno>=0 ){
238 + pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0);
239 + pIdx->aConstraintUsage[iPgno].omit = 1;
240 + pIdx->estimatedCost = 100;
241 + pIdx->estimatedRows = 50;
242 +
243 + if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){
244 + int iCol = pIdx->aOrderBy[0].iColumn;
245 + if( pIdx->nOrderBy==1 ){
246 + pIdx->orderByConsumed = (iCol==0 || iCol==1);
247 + }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){
248 + pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1);
249 + }
250 + }
251 +
252 + }else{
253 + pIdx->estimatedCost = 100000000;
254 + pIdx->estimatedRows = 1000000000;
255 + }
256 + pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00);
257 + return SQLITE_OK;
258 +}
259 +
260 +/*
261 +** Open a new sqlite_dbdata or sqlite_dbptr cursor.
262 +*/
263 +static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
264 + DbdataCursor *pCsr;
265 +
266 + pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor));
267 + if( pCsr==0 ){
268 + return SQLITE_NOMEM;
269 + }else{
270 + memset(pCsr, 0, sizeof(DbdataCursor));
271 + pCsr->base.pVtab = pVTab;
272 + }
273 +
274 + *ppCursor = (sqlite3_vtab_cursor *)pCsr;
275 + return SQLITE_OK;
276 +}
277 +
278 +/*
279 +** Restore a cursor object to the state it was in when first allocated
280 +** by dbdataOpen().
281 +*/
282 +static void dbdataResetCursor(DbdataCursor *pCsr){
283 + DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab);
284 + if( pTab->pStmt==0 ){
285 + pTab->pStmt = pCsr->pStmt;
286 + }else{
287 + sqlite3_finalize(pCsr->pStmt);
288 + }
289 + pCsr->pStmt = 0;
290 + pCsr->iPgno = 1;
291 + pCsr->iCell = 0;
292 + pCsr->iField = 0;
293 + pCsr->bOnePage = 0;
294 + sqlite3_free(pCsr->aPage);
295 + sqlite3_free(pCsr->pRec);
296 + pCsr->pRec = 0;
297 + pCsr->aPage = 0;
298 +}
299 +
300 +/*
301 +** Close an sqlite_dbdata or sqlite_dbptr cursor.
302 +*/
303 +static int dbdataClose(sqlite3_vtab_cursor *pCursor){
304 + DbdataCursor *pCsr = (DbdataCursor*)pCursor;
305 + dbdataResetCursor(pCsr);
306 + sqlite3_free(pCsr);
307 + return SQLITE_OK;
308 +}
309 +
310 +/*
311 +** Utility methods to decode 16 and 32-bit big-endian unsigned integers.
312 +*/
313 +static u32 get_uint16(unsigned char *a){
314 + return (a[0]<<8)|a[1];
315 +}
316 +static u32 get_uint32(unsigned char *a){
317 + return ((u32)a[0]<<24)
318 + | ((u32)a[1]<<16)
319 + | ((u32)a[2]<<8)
320 + | ((u32)a[3]);
321 +}
322 +
323 +/*
324 +** Load page pgno from the database via the sqlite_dbpage virtual table.
325 +** If successful, set (*ppPage) to point to a buffer containing the page
326 +** data, (*pnPage) to the size of that buffer in bytes and return
327 +** SQLITE_OK. In this case it is the responsibility of the caller to
328 +** eventually free the buffer using sqlite3_free().
329 +**
330 +** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and
331 +** return an SQLite error code.
332 +*/
333 +static int dbdataLoadPage(
334 + DbdataCursor *pCsr, /* Cursor object */
335 + u32 pgno, /* Page number of page to load */
336 + u8 **ppPage, /* OUT: pointer to page buffer */
337 + int *pnPage /* OUT: Size of (*ppPage) in bytes */
338 +){
339 + int rc2;
340 + int rc = SQLITE_OK;
341 + sqlite3_stmt *pStmt = pCsr->pStmt;
342 +
343 + *ppPage = 0;
344 + *pnPage = 0;
345 + if( pgno>0 ){
346 + sqlite3_bind_int64(pStmt, 2, pgno);
347 + if( SQLITE_ROW==sqlite3_step(pStmt) ){
348 + int nCopy = sqlite3_column_bytes(pStmt, 0);
349 + if( nCopy>0 ){
350 + u8 *pPage;
351 + pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
352 + if( pPage==0 ){
353 + rc = SQLITE_NOMEM;
354 + }else{
355 + const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
356 + memcpy(pPage, pCopy, nCopy);
357 + memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
358 + }
359 + *ppPage = pPage;
360 + *pnPage = nCopy;
361 + }
362 + }
363 + rc2 = sqlite3_reset(pStmt);
364 + if( rc==SQLITE_OK ) rc = rc2;
365 + }
366 +
367 + return rc;
368 +}
369 +
370 +/*
371 +** Read a varint. Put the value in *pVal and return the number of bytes.
372 +*/
373 +static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){
374 + sqlite3_uint64 u = 0;
375 + int i;
376 + for(i=0; i<8; i++){
377 + u = (u<<7) + (z[i]&0x7f);
378 + if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
379 + }
380 + u = (u<<8) + (z[i]&0xff);
381 + *pVal = (sqlite3_int64)u;
382 + return 9;
383 +}
384 +
385 +/*
386 +** Like dbdataGetVarint(), but set the output to 0 if it is less than 0
387 +** or greater than 0xFFFFFFFF. This can be used for all varints in an
388 +** SQLite database except for key values in intkey tables.
389 +*/
390 +static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){
391 + sqlite3_int64 val;
392 + int nRet = dbdataGetVarint(z, &val);
393 + if( val<0 || val>0xFFFFFFFF ) val = 0;
394 + *pVal = val;
395 + return nRet;
396 +}
397 +
398 +/*
399 +** Return the number of bytes of space used by an SQLite value of type
400 +** eType.
401 +*/
402 +static int dbdataValueBytes(int eType){
403 + switch( eType ){
404 + case 0: case 8: case 9:
405 + case 10: case 11:
406 + return 0;
407 + case 1:
408 + return 1;
409 + case 2:
410 + return 2;
411 + case 3:
412 + return 3;
413 + case 4:
414 + return 4;
415 + case 5:
416 + return 6;
417 + case 6:
418 + case 7:
419 + return 8;
420 + default:
421 + if( eType>0 ){
422 + return ((eType-12) / 2);
423 + }
424 + return 0;
425 + }
426 +}
427 +
428 +/*
429 +** Load a value of type eType from buffer pData and use it to set the
430 +** result of context object pCtx.
431 +*/
432 +static void dbdataValue(
433 + sqlite3_context *pCtx,
434 + u32 enc,
435 + int eType,
436 + u8 *pData,
437 + sqlite3_int64 nData
438 +){
439 + if( eType>=0 && dbdataValueBytes(eType)<=nData ){
440 + switch( eType ){
441 + case 0:
442 + case 10:
443 + case 11:
444 + sqlite3_result_null(pCtx);
445 + break;
446 +
447 + case 8:
448 + sqlite3_result_int(pCtx, 0);
449 + break;
450 + case 9:
451 + sqlite3_result_int(pCtx, 1);
452 + break;
453 +
454 + case 1: case 2: case 3: case 4: case 5: case 6: case 7: {
455 + sqlite3_uint64 v = (signed char)pData[0];
456 + pData++;
457 + switch( eType ){
458 + case 7:
459 + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2;
460 + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2;
461 + case 4: v = (v<<8) + pData[0]; pData++;
462 + case 3: v = (v<<8) + pData[0]; pData++;
463 + case 2: v = (v<<8) + pData[0]; pData++;
464 + }
465 +
466 + if( eType==7 ){
467 + double r;
468 + memcpy(&r, &v, sizeof(r));
469 + sqlite3_result_double(pCtx, r);
470 + }else{
471 + sqlite3_result_int64(pCtx, (sqlite3_int64)v);
472 + }
473 + break;
474 + }
475 +
476 + default: {
477 + int n = ((eType-12) / 2);
478 + if( eType % 2 ){
479 + switch( enc ){
480 +#ifndef SQLITE_OMIT_UTF16
481 + case SQLITE_UTF16BE:
482 + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
483 + break;
484 + case SQLITE_UTF16LE:
485 + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
486 + break;
487 +#endif
488 + default:
489 + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT);
490 + break;
491 + }
492 + }else{
493 + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
494 + }
495 + }
496 + }
497 + }
498 +}
499 +
500 +/*
501 +** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry.
502 +*/
503 +static int dbdataNext(sqlite3_vtab_cursor *pCursor){
504 + DbdataCursor *pCsr = (DbdataCursor*)pCursor;
505 + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
506 +
507 + pCsr->iRowid++;
508 + while( 1 ){
509 + int rc;
510 + int iOff = (pCsr->iPgno==1 ? 100 : 0);
511 + int bNextPage = 0;
512 +
513 + if( pCsr->aPage==0 ){
514 + while( 1 ){
515 + if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK;
516 + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage);
517 + if( rc!=SQLITE_OK ) return rc;
518 + if( pCsr->aPage && pCsr->nPage>=256 ) break;
519 + sqlite3_free(pCsr->aPage);
520 + pCsr->aPage = 0;
521 + if( pCsr->bOnePage ) return SQLITE_OK;
522 + pCsr->iPgno++;
523 + }
524 +
525 + assert( iOff+3+2<=pCsr->nPage );
526 + pCsr->iCell = pTab->bPtr ? -2 : 0;
527 + pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]);
528 + }
529 +
530 + if( pTab->bPtr ){
531 + if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){
532 + pCsr->iCell = pCsr->nCell;
533 + }
534 + pCsr->iCell++;
535 + if( pCsr->iCell>=pCsr->nCell ){
536 + sqlite3_free(pCsr->aPage);
537 + pCsr->aPage = 0;
538 + if( pCsr->bOnePage ) return SQLITE_OK;
539 + pCsr->iPgno++;
540 + }else{
541 + return SQLITE_OK;
542 + }
543 + }else{
544 + /* If there is no record loaded, load it now. */
545 + if( pCsr->pRec==0 ){
546 + int bHasRowid = 0;
547 + int nPointer = 0;
548 + sqlite3_int64 nPayload = 0;
549 + sqlite3_int64 nHdr = 0;
550 + int iHdr;
551 + int U, X;
552 + int nLocal;
553 +
554 + switch( pCsr->aPage[iOff] ){
555 + case 0x02:
556 + nPointer = 4;
557 + break;
558 + case 0x0a:
559 + break;
560 + case 0x0d:
561 + bHasRowid = 1;
562 + break;
563 + default:
564 + /* This is not a b-tree page with records on it. Continue. */
565 + pCsr->iCell = pCsr->nCell;
566 + break;
567 + }
568 +
569 + if( pCsr->iCell>=pCsr->nCell ){
570 + bNextPage = 1;
571 + }else{
572 +
573 + iOff += 8 + nPointer + pCsr->iCell*2;
574 + if( iOff>pCsr->nPage ){
575 + bNextPage = 1;
576 + }else{
577 + iOff = get_uint16(&pCsr->aPage[iOff]);
578 + }
579 +
580 + /* For an interior node cell, skip past the child-page number */
581 + iOff += nPointer;
582 +
583 + /* Load the "byte of payload including overflow" field */
584 + if( bNextPage || iOff>pCsr->nPage ){
585 + bNextPage = 1;
586 + }else{
587 + iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
588 + }
589 +
590 + /* If this is a leaf intkey cell, load the rowid */
591 + if( bHasRowid && !bNextPage && iOff<pCsr->nPage ){
592 + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey);
593 + }
594 +
595 + /* Figure out how much data to read from the local page */
596 + U = pCsr->nPage;
597 + if( bHasRowid ){
598 + X = U-35;
599 + }else{
600 + X = ((U-12)*64/255)-23;
601 + }
602 + if( nPayload<=X ){
603 + nLocal = nPayload;
604 + }else{
605 + int M, K;
606 + M = ((U-12)*32/255)-23;
607 + K = M+((nPayload-M)%(U-4));
608 + if( K<=X ){
609 + nLocal = K;
610 + }else{
611 + nLocal = M;
612 + }
613 + }
614 +
615 + if( bNextPage || nLocal+iOff>pCsr->nPage ){
616 + bNextPage = 1;
617 + }else{
618 +
619 + /* Allocate space for payload. And a bit more to catch small buffer
620 + ** overruns caused by attempting to read a varint or similar from
621 + ** near the end of a corrupt record. */
622 + pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES);
623 + if( pCsr->pRec==0 ) return SQLITE_NOMEM;
624 + memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES);
625 + pCsr->nRec = nPayload;
626 +
627 + /* Load the nLocal bytes of payload */
628 + memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal);
629 + iOff += nLocal;
630 +
631 + /* Load content from overflow pages */
632 + if( nPayload>nLocal ){
633 + sqlite3_int64 nRem = nPayload - nLocal;
634 + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
635 + while( nRem>0 ){
636 + u8 *aOvfl = 0;
637 + int nOvfl = 0;
638 + int nCopy;
639 + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl);
640 + assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage );
641 + if( rc!=SQLITE_OK ) return rc;
642 + if( aOvfl==0 ) break;
643 +
644 + nCopy = U-4;
645 + if( nCopy>nRem ) nCopy = nRem;
646 + memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy);
647 + nRem -= nCopy;
648 +
649 + pgnoOvfl = get_uint32(aOvfl);
650 + sqlite3_free(aOvfl);
651 + }
652 + }
653 +
654 + iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr);
655 + if( nHdr>nPayload ) nHdr = 0;
656 + pCsr->nHdr = nHdr;
657 + pCsr->pHdrPtr = &pCsr->pRec[iHdr];
658 + pCsr->pPtr = &pCsr->pRec[pCsr->nHdr];
659 + pCsr->iField = (bHasRowid ? -1 : 0);
660 + }
661 + }
662 + }else{
663 + pCsr->iField++;
664 + if( pCsr->iField>0 ){
665 + sqlite3_int64 iType;
666 + if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){
667 + bNextPage = 1;
668 + }else{
669 + int szField = 0;
670 + pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
671 + szField = dbdataValueBytes(iType);
672 + if( (pCsr->nRec - (pCsr->pPtr - pCsr->pRec))<szField ){
673 + pCsr->pPtr = &pCsr->pRec[pCsr->nRec];
674 + }else{
675 + pCsr->pPtr += szField;
676 + }
677 + }
678 + }
679 + }
680 +
681 + if( bNextPage ){
682 + sqlite3_free(pCsr->aPage);
683 + sqlite3_free(pCsr->pRec);
684 + pCsr->aPage = 0;
685 + pCsr->pRec = 0;
686 + if( pCsr->bOnePage ) return SQLITE_OK;
687 + pCsr->iPgno++;
688 + }else{
689 + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){
690 + return SQLITE_OK;
691 + }
692 +
693 + /* Advance to the next cell. The next iteration of the loop will load
694 + ** the record and so on. */
695 + sqlite3_free(pCsr->pRec);
696 + pCsr->pRec = 0;
697 + pCsr->iCell++;
698 + }
699 + }
700 + }
701 +
702 + assert( !"can't get here" );
703 + return SQLITE_OK;
704 +}
705 +
706 +/*
707 +** Return true if the cursor is at EOF.
708 +*/
709 +static int dbdataEof(sqlite3_vtab_cursor *pCursor){
710 + DbdataCursor *pCsr = (DbdataCursor*)pCursor;
711 + return pCsr->aPage==0;
712 +}
713 +
714 +/*
715 +** Return true if nul-terminated string zSchema ends in "()". Or false
716 +** otherwise.
717 +*/
718 +static int dbdataIsFunction(const char *zSchema){
719 + size_t n = strlen(zSchema);
720 + if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){
721 + return (int)n-2;
722 + }
723 + return 0;
724 +}
725 +
726 +/*
727 +** Determine the size in pages of database zSchema (where zSchema is
728 +** "main", "temp" or the name of an attached database) and set
729 +** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise,
730 +** an SQLite error code.
731 +*/
732 +static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
733 + DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab;
734 + char *zSql = 0;
735 + int rc, rc2;
736 + int nFunc = 0;
737 + sqlite3_stmt *pStmt = 0;
738 +
739 + if( (nFunc = dbdataIsFunction(zSchema))>0 ){
740 + zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema);
741 + }else{
742 + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
743 + }
744 + if( zSql==0 ) return SQLITE_NOMEM;
745 +
746 + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
747 + sqlite3_free(zSql);
748 + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
749 + pCsr->szDb = sqlite3_column_int(pStmt, 0);
750 + }
751 + rc2 = sqlite3_finalize(pStmt);
752 + if( rc==SQLITE_OK ) rc = rc2;
753 + return rc;
754 +}
755 +
756 +/*
757 +** Attempt to figure out the encoding of the database by retrieving page 1
758 +** and inspecting the header field. If successful, set the pCsr->enc variable
759 +** and return SQLITE_OK. Otherwise, return an SQLite error code.
760 +*/
761 +static int dbdataGetEncoding(DbdataCursor *pCsr){
762 + int rc = SQLITE_OK;
763 + int nPg1 = 0;
764 + u8 *aPg1 = 0;
765 + rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1);
766 + if( rc==SQLITE_OK && nPg1>=(56+4) ){
767 + pCsr->enc = get_uint32(&aPg1[56]);
768 + }
769 + sqlite3_free(aPg1);
770 + return rc;
771 +}
772 +
773 +
774 +/*
775 +** xFilter method for sqlite_dbdata and sqlite_dbptr.
776 +*/
777 +static int dbdataFilter(
778 + sqlite3_vtab_cursor *pCursor,
779 + int idxNum, const char *idxStr,
780 + int argc, sqlite3_value **argv
781 +){
782 + DbdataCursor *pCsr = (DbdataCursor*)pCursor;
783 + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
784 + int rc = SQLITE_OK;
785 + const char *zSchema = "main";
786 + (void)idxStr;
787 + (void)argc;
788 +
789 + dbdataResetCursor(pCsr);
790 + assert( pCsr->iPgno==1 );
791 + if( idxNum & 0x01 ){
792 + zSchema = (const char*)sqlite3_value_text(argv[0]);
793 + if( zSchema==0 ) zSchema = "";
794 + }
795 + if( idxNum & 0x02 ){
796 + pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
797 + pCsr->bOnePage = 1;
798 + }else{
799 + rc = dbdataDbsize(pCsr, zSchema);
800 + }
801 +
802 + if( rc==SQLITE_OK ){
803 + int nFunc = 0;
804 + if( pTab->pStmt ){
805 + pCsr->pStmt = pTab->pStmt;
806 + pTab->pStmt = 0;
807 + }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){
808 + char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema);
809 + if( zSql==0 ){
810 + rc = SQLITE_NOMEM;
811 + }else{
812 + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
813 + sqlite3_free(zSql);
814 + }
815 + }else{
816 + rc = sqlite3_prepare_v2(pTab->db,
817 + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
818 + &pCsr->pStmt, 0
819 + );
820 + }
821 + }
822 + if( rc==SQLITE_OK ){
823 + rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT);
824 + }
825 +
826 + /* Try to determine the encoding of the db by inspecting the header
827 + ** field on page 1. */
828 + if( rc==SQLITE_OK ){
829 + rc = dbdataGetEncoding(pCsr);
830 + }
831 +
832 + if( rc!=SQLITE_OK ){
833 + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
834 + }
835 +
836 + if( rc==SQLITE_OK ){
837 + rc = dbdataNext(pCursor);
838 + }
839 + return rc;
840 +}
841 +
842 +/*
843 +** Return a column for the sqlite_dbdata or sqlite_dbptr table.
844 +*/
845 +static int dbdataColumn(
846 + sqlite3_vtab_cursor *pCursor,
847 + sqlite3_context *ctx,
848 + int i
849 +){
850 + DbdataCursor *pCsr = (DbdataCursor*)pCursor;
851 + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
852 + if( pTab->bPtr ){
853 + switch( i ){
854 + case DBPTR_COLUMN_PGNO:
855 + sqlite3_result_int64(ctx, pCsr->iPgno);
856 + break;
857 + case DBPTR_COLUMN_CHILD: {
858 + int iOff = pCsr->iPgno==1 ? 100 : 0;
859 + if( pCsr->iCell<0 ){
860 + iOff += 8;
861 + }else{
862 + iOff += 12 + pCsr->iCell*2;
863 + if( iOff>pCsr->nPage ) return SQLITE_OK;
864 + iOff = get_uint16(&pCsr->aPage[iOff]);
865 + }
866 + if( iOff<=pCsr->nPage ){
867 + sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff]));
868 + }
869 + break;
870 + }
871 + }
872 + }else{
873 + switch( i ){
874 + case DBDATA_COLUMN_PGNO:
875 + sqlite3_result_int64(ctx, pCsr->iPgno);
876 + break;
877 + case DBDATA_COLUMN_CELL:
878 + sqlite3_result_int(ctx, pCsr->iCell);
879 + break;
880 + case DBDATA_COLUMN_FIELD:
881 + sqlite3_result_int(ctx, pCsr->iField);
882 + break;
883 + case DBDATA_COLUMN_VALUE: {
884 + if( pCsr->iField<0 ){
885 + sqlite3_result_int64(ctx, pCsr->iIntkey);
886 + }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){
887 + sqlite3_int64 iType;
888 + dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
889 + dbdataValue(
890 + ctx, pCsr->enc, iType, pCsr->pPtr,
891 + &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
892 + );
893 + }
894 + break;
895 + }
896 + }
897 + }
898 + return SQLITE_OK;
899 +}
900 +
901 +/*
902 +** Return the rowid for an sqlite_dbdata or sqlite_dptr table.
903 +*/
904 +static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
905 + DbdataCursor *pCsr = (DbdataCursor*)pCursor;
906 + *pRowid = pCsr->iRowid;
907 + return SQLITE_OK;
908 +}
909 +
910 +
911 +/*
912 +** Invoke this routine to register the "sqlite_dbdata" virtual table module
913 +*/
914 +static int sqlite3DbdataRegister(sqlite3 *db){
915 + static sqlite3_module dbdata_module = {
916 + 0, /* iVersion */
917 + 0, /* xCreate */
918 + dbdataConnect, /* xConnect */
919 + dbdataBestIndex, /* xBestIndex */
920 + dbdataDisconnect, /* xDisconnect */
921 + 0, /* xDestroy */
922 + dbdataOpen, /* xOpen - open a cursor */
923 + dbdataClose, /* xClose - close a cursor */
924 + dbdataFilter, /* xFilter - configure scan constraints */
925 + dbdataNext, /* xNext - advance a cursor */
926 + dbdataEof, /* xEof - check for end of scan */
927 + dbdataColumn, /* xColumn - read data */
928 + dbdataRowid, /* xRowid - read data */
929 + 0, /* xUpdate */
930 + 0, /* xBegin */
931 + 0, /* xSync */
932 + 0, /* xCommit */
933 + 0, /* xRollback */
934 + 0, /* xFindMethod */
935 + 0, /* xRename */
936 + 0, /* xSavepoint */
937 + 0, /* xRelease */
938 + 0, /* xRollbackTo */
939 + 0 /* xShadowName */
940 + };
941 +
942 + int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0);
943 + if( rc==SQLITE_OK ){
944 + rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1);
945 + }
946 + return rc;
947 +}
948 +
949 +int sqlite3_dbdata_init(
950 + sqlite3 *db,
951 + char **pzErrMsg,
952 + const sqlite3_api_routines *pApi
953 +){
954 + (void)pzErrMsg;
955 + return sqlite3DbdataRegister(db);
956 +}
957 +
958 +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
959 +#pragma GCC diagnostic pop
database/sqlite/sqlite3recover.c new
+2872
@@ -0,0 +1,2872 @@
1 +/*
2 +** 2022-08-27
3 +**
4 +** The author disclaims copyright to this source code. In place of
5 +** a legal notice, here is a blessing:
6 +**
7 +** May you do good and not evil.
8 +** May you find forgiveness for yourself and forgive others.
9 +** May you share freely, never taking more than you give.
10 +**
11 +*************************************************************************
12 +**
13 +*/
14 +
15 +#pragma GCC diagnostic push
16 +#pragma GCC diagnostic ignored "-Wsign-compare"
17 +#include "sqlite3recover.h"
18 +#include <assert.h>
19 +#include <string.h>
20 +
21 +#ifndef SQLITE_OMIT_VIRTUALTABLE
22 +
23 +/*
24 +** Declaration for public API function in file dbdata.c. This may be called
25 +** with NULL as the final two arguments to register the sqlite_dbptr and
26 +** sqlite_dbdata virtual tables with a database handle.
27 +*/
28 +#ifdef _WIN32
29 +__declspec(dllexport)
30 +#endif
31 +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*);
32 +
33 +typedef unsigned int u32;
34 +typedef unsigned char u8;
35 +typedef sqlite3_int64 i64;
36 +
37 +typedef struct RecoverTable RecoverTable;
38 +typedef struct RecoverColumn RecoverColumn;
39 +
40 +/*
41 +** When recovering rows of data that can be associated with table
42 +** definitions recovered from the sqlite_schema table, each table is
43 +** represented by an instance of the following object.
44 +**
45 +** iRoot:
46 +** The root page in the original database. Not necessarily (and usually
47 +** not) the same in the recovered database.
48 +**
49 +** zTab:
50 +** Name of the table.
51 +**
52 +** nCol/aCol[]:
53 +** aCol[] is an array of nCol columns. In the order in which they appear
54 +** in the table.
55 +**
56 +** bIntkey:
57 +** Set to true for intkey tables, false for WITHOUT ROWID.
58 +**
59 +** iRowidBind:
60 +** Each column in the aCol[] array has associated with it the index of
61 +** the bind parameter its values will be bound to in the INSERT statement
62 +** used to construct the output database. If the table does has a rowid
63 +** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the
64 +** index of the bind paramater to which the rowid value should be bound.
65 +** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY
66 +** KEY column, then the rowid value should be bound to the index associated
67 +** with the column.
68 +**
69 +** pNext:
70 +** All RecoverTable objects used by the recovery operation are allocated
71 +** and populated as part of creating the recovered database schema in
72 +** the output database, before any non-schema data are recovered. They
73 +** are then stored in a singly-linked list linked by this variable beginning
74 +** at sqlite3_recover.pTblList.
75 +*/
76 +struct RecoverTable {
77 + u32 iRoot; /* Root page in original database */
78 + char *zTab; /* Name of table */
79 + int nCol; /* Number of columns in table */
80 + RecoverColumn *aCol; /* Array of columns */
81 + int bIntkey; /* True for intkey, false for without rowid */
82 + int iRowidBind; /* If >0, bind rowid to INSERT here */
83 + RecoverTable *pNext;
84 +};
85 +
86 +/*
87 +** Each database column is represented by an instance of the following object
88 +** stored in the RecoverTable.aCol[] array of the associated table.
89 +**
90 +** iField:
91 +** The index of the associated field within database records. Or -1 if
92 +** there is no associated field (e.g. for virtual generated columns).
93 +**
94 +** iBind:
95 +** The bind index of the INSERT statement to bind this columns values
96 +** to. Or 0 if there is no such index (iff (iField<0)).
97 +**
98 +** bIPK:
99 +** True if this is the INTEGER PRIMARY KEY column.
100 +**
101 +** zCol:
102 +** Name of column.
103 +**
104 +** eHidden:
105 +** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each).
106 +*/
107 +struct RecoverColumn {
108 + int iField; /* Field in record on disk */
109 + int iBind; /* Binding to use in INSERT */
110 + int bIPK; /* True for IPK column */
111 + char *zCol;
112 + int eHidden;
113 +};
114 +
115 +#define RECOVER_EHIDDEN_NONE 0 /* Normal database column */
116 +#define RECOVER_EHIDDEN_HIDDEN 1 /* Column is __HIDDEN__ */
117 +#define RECOVER_EHIDDEN_VIRTUAL 2 /* Virtual generated column */
118 +#define RECOVER_EHIDDEN_STORED 3 /* Stored generated column */
119 +
120 +/*
121 +** Bitmap object used to track pages in the input database. Allocated
122 +** and manipulated only by the following functions:
123 +**
124 +** recoverBitmapAlloc()
125 +** recoverBitmapFree()
126 +** recoverBitmapSet()
127 +** recoverBitmapQuery()
128 +**
129 +** nPg:
130 +** Largest page number that may be stored in the bitmap. The range
131 +** of valid keys is 1 to nPg, inclusive.
132 +**
133 +** aElem[]:
134 +** Array large enough to contain a bit for each key. For key value
135 +** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32].
136 +** In other words, the following is true if bit iKey is set, or
137 +** false if it is clear:
138 +**
139 +** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0
140 +*/
141 +typedef struct RecoverBitmap RecoverBitmap;
142 +struct RecoverBitmap {
143 + i64 nPg; /* Size of bitmap */
144 + u32 aElem[1]; /* Array of 32-bit bitmasks */
145 +};
146 +
147 +/*
148 +** State variables (part of the sqlite3_recover structure) used while
149 +** recovering data for tables identified in the recovered schema (state
150 +** RECOVER_STATE_WRITING).
151 +*/
152 +typedef struct RecoverStateW1 RecoverStateW1;
153 +struct RecoverStateW1 {
154 + sqlite3_stmt *pTbls;
155 + sqlite3_stmt *pSel;
156 + sqlite3_stmt *pInsert;
157 + int nInsert;
158 +
159 + RecoverTable *pTab; /* Table currently being written */
160 + int nMax; /* Max column count in any schema table */
161 + sqlite3_value **apVal; /* Array of nMax values */
162 + int nVal; /* Number of valid entries in apVal[] */
163 + int bHaveRowid;
164 + i64 iRowid;
165 + i64 iPrevPage;
166 + int iPrevCell;
167 +};
168 +
169 +/*
170 +** State variables (part of the sqlite3_recover structure) used while
171 +** recovering data destined for the lost and found table (states
172 +** RECOVER_STATE_LOSTANDFOUND[123]).
173 +*/
174 +typedef struct RecoverStateLAF RecoverStateLAF;
175 +struct RecoverStateLAF {
176 + RecoverBitmap *pUsed;
177 + i64 nPg; /* Size of db in pages */
178 + sqlite3_stmt *pAllAndParent;
179 + sqlite3_stmt *pMapInsert;
180 + sqlite3_stmt *pMaxField;
181 + sqlite3_stmt *pUsedPages;
182 + sqlite3_stmt *pFindRoot;
183 + sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */
184 + sqlite3_stmt *pAllPage;
185 + sqlite3_stmt *pPageData;
186 + sqlite3_value **apVal;
187 + int nMaxField;
188 +};
189 +
190 +/*
191 +** Main recover handle structure.
192 +*/
193 +struct sqlite3_recover {
194 + /* Copies of sqlite3_recover_init[_sql]() parameters */
195 + sqlite3 *dbIn; /* Input database */
196 + char *zDb; /* Name of input db ("main" etc.) */
197 + char *zUri; /* URI for output database */
198 + void *pSqlCtx; /* SQL callback context */
199 + int (*xSql)(void*,const char*); /* Pointer to SQL callback function */
200 +
201 + /* Values configured by sqlite3_recover_config() */
202 + char *zStateDb; /* State database to use (or NULL) */
203 + char *zLostAndFound; /* Name of lost-and-found table (or NULL) */
204 + int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */
205 + int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */
206 + int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */
207 +
208 + int pgsz;
209 + int detected_pgsz;
210 + int nReserve;
211 + u8 *pPage1Disk;
212 + u8 *pPage1Cache;
213 +
214 + /* Error code and error message */
215 + int errCode; /* For sqlite3_recover_errcode() */
216 + char *zErrMsg; /* For sqlite3_recover_errmsg() */
217 +
218 + int eState;
219 + int bCloseTransaction;
220 +
221 + /* Variables used with eState==RECOVER_STATE_WRITING */
222 + RecoverStateW1 w1;
223 +
224 + /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */
225 + RecoverStateLAF laf;
226 +
227 + /* Fields used within sqlite3_recover_run() */
228 + sqlite3 *dbOut; /* Output database */
229 + sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */
230 + RecoverTable *pTblList; /* List of tables recovered from schema */
231 +};
232 +
233 +/*
234 +** The various states in which an sqlite3_recover object may exist:
235 +**
236 +** RECOVER_STATE_INIT:
237 +** The object is initially created in this state. sqlite3_recover_step()
238 +** has yet to be called. This is the only state in which it is permitted
239 +** to call sqlite3_recover_config().
240 +**
241 +** RECOVER_STATE_WRITING:
242 +**
243 +** RECOVER_STATE_LOSTANDFOUND1:
244 +** State to populate the bitmap of pages used by other tables or the
245 +** database freelist.
246 +**
247 +** RECOVER_STATE_LOSTANDFOUND2:
248 +** Populate the recovery.map table - used to figure out a "root" page
249 +** for each lost page from in the database from which records are
250 +** extracted.
251 +**
252 +** RECOVER_STATE_LOSTANDFOUND3:
253 +** Populate the lost-and-found table itself.
254 +*/
255 +#define RECOVER_STATE_INIT 0
256 +#define RECOVER_STATE_WRITING 1
257 +#define RECOVER_STATE_LOSTANDFOUND1 2
258 +#define RECOVER_STATE_LOSTANDFOUND2 3
259 +#define RECOVER_STATE_LOSTANDFOUND3 4
260 +#define RECOVER_STATE_SCHEMA2 5
261 +#define RECOVER_STATE_DONE 6
262 +
263 +
264 +/*
265 +** Global variables used by this extension.
266 +*/
267 +typedef struct RecoverGlobal RecoverGlobal;
268 +struct RecoverGlobal {
269 + const sqlite3_io_methods *pMethods;
270 + sqlite3_recover *p;
271 +};
272 +static RecoverGlobal recover_g;
273 +
274 +/*
275 +** Use this static SQLite mutex to protect the globals during the
276 +** first call to sqlite3_recover_step().
277 +*/
278 +#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2
279 +
280 +
281 +/*
282 +** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid).
283 +*/
284 +#define RECOVER_ROWID_DEFAULT 1
285 +
286 +/*
287 +** Mutex handling:
288 +**
289 +** recoverEnterMutex() - Enter the recovery mutex
290 +** recoverLeaveMutex() - Leave the recovery mutex
291 +** recoverAssertMutexHeld() - Assert that the recovery mutex is held
292 +*/
293 +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0
294 +# define recoverEnterMutex()
295 +# define recoverLeaveMutex()
296 +#else
297 +static void recoverEnterMutex(void){
298 + sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID));
299 +}
300 +static void recoverLeaveMutex(void){
301 + sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID));
302 +}
303 +#endif
304 +#if SQLITE_THREADSAFE+0>=1 && defined(SQLITE_DEBUG)
305 +static void recoverAssertMutexHeld(void){
306 + assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) );
307 +}
308 +#else
309 +# define recoverAssertMutexHeld()
310 +#endif
311 +
312 +
313 +/*
314 +** Like strlen(). But handles NULL pointer arguments.
315 +*/
316 +static int recoverStrlen(const char *zStr){
317 + if( zStr==0 ) return 0;
318 + return (int)(strlen(zStr)&0x7fffffff);
319 +}
320 +
321 +/*
322 +** This function is a no-op if the recover handle passed as the first
323 +** argument already contains an error (if p->errCode!=SQLITE_OK).
324 +**
325 +** Otherwise, an attempt is made to allocate, zero and return a buffer nByte
326 +** bytes in size. If successful, a pointer to the new buffer is returned. Or,
327 +** if an OOM error occurs, NULL is returned and the handle error code
328 +** (p->errCode) set to SQLITE_NOMEM.
329 +*/
330 +static void *recoverMalloc(sqlite3_recover *p, i64 nByte){
331 + void *pRet = 0;
332 + assert( nByte>0 );
333 + if( p->errCode==SQLITE_OK ){
334 + pRet = sqlite3_malloc64(nByte);
335 + if( pRet ){
336 + memset(pRet, 0, nByte);
337 + }else{
338 + p->errCode = SQLITE_NOMEM;
339 + }
340 + }
341 + return pRet;
342 +}
343 +
344 +/*
345 +** Set the error code and error message for the recover handle passed as
346 +** the first argument. The error code is set to the value of parameter
347 +** errCode.
348 +**
349 +** Parameter zFmt must be a printf() style formatting string. The handle
350 +** error message is set to the result of using any trailing arguments for
351 +** parameter substitutions in the formatting string.
352 +**
353 +** For example:
354 +**
355 +** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename);
356 +*/
357 +static int recoverError(
358 + sqlite3_recover *p,
359 + int errCode,
360 + const char *zFmt, ...
361 +){
362 + char *z = 0;
363 + va_list ap;
364 + va_start(ap, zFmt);
365 + if( zFmt ){
366 + z = sqlite3_vmprintf(zFmt, ap);
367 + va_end(ap);
368 + }
369 + sqlite3_free(p->zErrMsg);
370 + p->zErrMsg = z;
371 + p->errCode = errCode;
372 + return errCode;
373 +}
374 +
375 +
376 +/*
377 +** This function is a no-op if p->errCode is initially other than SQLITE_OK.
378 +** In this case it returns NULL.
379 +**
380 +** Otherwise, an attempt is made to allocate and return a bitmap object
381 +** large enough to store a bit for all page numbers between 1 and nPg,
382 +** inclusive. The bitmap is initially zeroed.
383 +*/
384 +static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){
385 + int nElem = (nPg+1+31) / 32;
386 + int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32);
387 + RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte);
388 +
389 + if( pRet ){
390 + pRet->nPg = nPg;
391 + }
392 + return pRet;
393 +}
394 +
395 +/*
396 +** Free a bitmap object allocated by recoverBitmapAlloc().
397 +*/
398 +static void recoverBitmapFree(RecoverBitmap *pMap){
399 + sqlite3_free(pMap);
400 +}
401 +
402 +/*
403 +** Set the bit associated with page iPg in bitvec pMap.
404 +*/
405 +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){
406 + if( iPg<=pMap->nPg ){
407 + int iElem = (iPg / 32);
408 + int iBit = (iPg % 32);
409 + pMap->aElem[iElem] |= (((u32)1) << iBit);
410 + }
411 +}
412 +
413 +/*
414 +** Query bitmap object pMap for the state of the bit associated with page
415 +** iPg. Return 1 if it is set, or 0 otherwise.
416 +*/
417 +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){
418 + int ret = 1;
419 + if( iPg<=pMap->nPg && iPg>0 ){
420 + int iElem = (iPg / 32);
421 + int iBit = (iPg % 32);
422 + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0;
423 + }
424 + return ret;
425 +}
426 +
427 +/*
428 +** Set the recover handle error to the error code and message returned by
429 +** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database
430 +** handle db.
431 +*/
432 +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){
433 + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db));
434 +}
435 +
436 +/*
437 +** This function is a no-op if recover handle p already contains an error
438 +** (if p->errCode!=SQLITE_OK).
439 +**
440 +** Otherwise, it attempts to prepare the SQL statement in zSql against
441 +** database handle db. If successful, the statement handle is returned.
442 +** Or, if an error occurs, NULL is returned and an error left in the
443 +** recover handle.
444 +*/
445 +static sqlite3_stmt *recoverPrepare(
446 + sqlite3_recover *p,
447 + sqlite3 *db,
448 + const char *zSql
449 +){
450 + sqlite3_stmt *pStmt = 0;
451 + if( p->errCode==SQLITE_OK ){
452 + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){
453 + recoverDbError(p, db);
454 + }
455 + }
456 + return pStmt;
457 +}
458 +
459 +/*
460 +** This function is a no-op if recover handle p already contains an error
461 +** (if p->errCode!=SQLITE_OK).
462 +**
463 +** Otherwise, argument zFmt is used as a printf() style format string,
464 +** along with any trailing arguments, to create an SQL statement. This
465 +** SQL statement is prepared against database handle db and, if successful,
466 +** the statment handle returned. Or, if an error occurs - either during
467 +** the printf() formatting or when preparing the resulting SQL - an
468 +** error code and message are left in the recover handle.
469 +*/
470 +static sqlite3_stmt *recoverPreparePrintf(
471 + sqlite3_recover *p,
472 + sqlite3 *db,
473 + const char *zFmt, ...
474 +){
475 + sqlite3_stmt *pStmt = 0;
476 + if( p->errCode==SQLITE_OK ){
477 + va_list ap;
478 + char *z;
479 + va_start(ap, zFmt);
480 + z = sqlite3_vmprintf(zFmt, ap);
481 + va_end(ap);
482 + if( z==0 ){
483 + p->errCode = SQLITE_NOMEM;
484 + }else{
485 + pStmt = recoverPrepare(p, db, z);
486 + sqlite3_free(z);
487 + }
488 + }
489 + return pStmt;
490 +}
491 +
492 +/*
493 +** Reset SQLite statement handle pStmt. If the call to sqlite3_reset()
494 +** indicates that an error occurred, and there is not already an error
495 +** in the recover handle passed as the first argument, set the error
496 +** code and error message appropriately.
497 +**
498 +** This function returns a copy of the statement handle pointer passed
499 +** as the second argument.
500 +*/
501 +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){
502 + int rc = sqlite3_reset(pStmt);
503 + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){
504 + recoverDbError(p, sqlite3_db_handle(pStmt));
505 + }
506 + return pStmt;
507 +}
508 +
509 +/*
510 +** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset()
511 +** indicates that an error occurred, and there is not already an error
512 +** in the recover handle passed as the first argument, set the error
513 +** code and error message appropriately.
514 +*/
515 +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){
516 + sqlite3 *db = sqlite3_db_handle(pStmt);
517 + int rc = sqlite3_finalize(pStmt);
518 + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){
519 + recoverDbError(p, db);
520 + }
521 +}
522 +
523 +/*
524 +** This function is a no-op if recover handle p already contains an error
525 +** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this
526 +** case.
527 +**
528 +** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK.
529 +** Or, if an error occurs, leave an error code and message in the recover
530 +** handle and return a copy of the error code.
531 +*/
532 +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){
533 + if( p->errCode==SQLITE_OK ){
534 + int rc = sqlite3_exec(db, zSql, 0, 0, 0);
535 + if( rc ){
536 + recoverDbError(p, db);
537 + }
538 + }
539 + return p->errCode;
540 +}
541 +
542 +/*
543 +** Bind the value pVal to parameter iBind of statement pStmt. Leave an
544 +** error in the recover handle passed as the first argument if an error
545 +** (e.g. an OOM) occurs.
546 +*/
547 +static void recoverBindValue(
548 + sqlite3_recover *p,
549 + sqlite3_stmt *pStmt,
550 + int iBind,
551 + sqlite3_value *pVal
552 +){
553 + if( p->errCode==SQLITE_OK ){
554 + int rc = sqlite3_bind_value(pStmt, iBind, pVal);
555 + if( rc ) recoverError(p, rc, 0);
556 + }
557 +}
558 +
559 +/*
560 +** This function is a no-op if recover handle p already contains an error
561 +** (if p->errCode!=SQLITE_OK). NULL is returned in this case.
562 +**
563 +** Otherwise, an attempt is made to interpret zFmt as a printf() style
564 +** formatting string and the result of using the trailing arguments for
565 +** parameter substitution with it written into a buffer obtained from
566 +** sqlite3_malloc(). If successful, a pointer to the buffer is returned.
567 +** It is the responsibility of the caller to eventually free the buffer
568 +** using sqlite3_free().
569 +**
570 +** Or, if an error occurs, an error code and message is left in the recover
571 +** handle and NULL returned.
572 +*/
573 +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){
574 + va_list ap;
575 + char *z;
576 + va_start(ap, zFmt);
577 + z = sqlite3_vmprintf(zFmt, ap);
578 + va_end(ap);
579 + if( p->errCode==SQLITE_OK ){
580 + if( z==0 ) p->errCode = SQLITE_NOMEM;
581 + }else{
582 + sqlite3_free(z);
583 + z = 0;
584 + }
585 + return z;
586 +}
587 +
588 +/*
589 +** This function is a no-op if recover handle p already contains an error
590 +** (if p->errCode!=SQLITE_OK). Zero is returned in this case.
591 +**
592 +** Otherwise, execute "PRAGMA page_count" against the input database. If
593 +** successful, return the integer result. Or, if an error occurs, leave an
594 +** error code and error message in the sqlite3_recover handle and return
595 +** zero.
596 +*/
597 +static i64 recoverPageCount(sqlite3_recover *p){
598 + i64 nPg = 0;
599 + if( p->errCode==SQLITE_OK ){
600 + sqlite3_stmt *pStmt = 0;
601 + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb);
602 + if( pStmt ){
603 + sqlite3_step(pStmt);
604 + nPg = sqlite3_column_int64(pStmt, 0);
605 + }
606 + recoverFinalize(p, pStmt);
607 + }
608 + return nPg;
609 +}
610 +
611 +/*
612 +** Implementation of SQL scalar function "read_i32". The first argument to
613 +** this function must be a blob. The second a non-negative integer. This
614 +** function reads and returns a 32-bit big-endian integer from byte
615 +** offset (4*<arg2>) of the blob.
616 +**
617 +** SELECT read_i32(<blob>, <idx>)
618 +*/
619 +static void recoverReadI32(
620 + sqlite3_context *context,
621 + int argc,
622 + sqlite3_value **argv
623 +){
624 + const unsigned char *pBlob;
625 + int nBlob;
626 + int iInt;
627 +
628 + assert( argc==2 );
629 + nBlob = sqlite3_value_bytes(argv[0]);
630 + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]);
631 + iInt = sqlite3_value_int(argv[1]) & 0xFFFF;
632 +
633 + if( (iInt+1)*4<=nBlob ){
634 + const unsigned char *a = &pBlob[iInt*4];
635 + i64 iVal = ((i64)a[0]<<24)
636 + + ((i64)a[1]<<16)
637 + + ((i64)a[2]<< 8)
638 + + ((i64)a[3]<< 0);
639 + sqlite3_result_int64(context, iVal);
640 + }
641 +}
642 +
643 +/*
644 +** Implementation of SQL scalar function "page_is_used". This function
645 +** is used as part of the procedure for locating orphan rows for the
646 +** lost-and-found table, and it depends on those routines having populated
647 +** the sqlite3_recover.laf.pUsed variable.
648 +**
649 +** The only argument to this function is a page-number. It returns true
650 +** if the page has already been used somehow during data recovery, or false
651 +** otherwise.
652 +**
653 +** SELECT page_is_used(<pgno>);
654 +*/
655 +static void recoverPageIsUsed(
656 + sqlite3_context *pCtx,
657 + int nArg,
658 + sqlite3_value **apArg
659 +){
660 + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
661 + i64 pgno = sqlite3_value_int64(apArg[0]);
662 + assert( nArg==1 );
663 + sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno));
664 +}
665 +
666 +/*
667 +** The implementation of a user-defined SQL function invoked by the
668 +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages
669 +** of the database being recovered.
670 +**
671 +** This function always takes a single integer argument. If the argument
672 +** is zero, then the value returned is the number of pages in the db being
673 +** recovered. If the argument is greater than zero, it is a page number.
674 +** The value returned in this case is an SQL blob containing the data for
675 +** the identified page of the db being recovered. e.g.
676 +**
677 +** SELECT getpage(0); -- return number of pages in db
678 +** SELECT getpage(4); -- return page 4 of db as a blob of data
679 +*/
680 +static void recoverGetPage(
681 + sqlite3_context *pCtx,
682 + int nArg,
683 + sqlite3_value **apArg
684 +){
685 + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
686 + i64 pgno = sqlite3_value_int64(apArg[0]);
687 + sqlite3_stmt *pStmt = 0;
688 +
689 + assert( nArg==1 );
690 + if( pgno==0 ){
691 + i64 nPg = recoverPageCount(p);
692 + sqlite3_result_int64(pCtx, nPg);
693 + return;
694 + }else{
695 + if( p->pGetPage==0 ){
696 + pStmt = p->pGetPage = recoverPreparePrintf(
697 + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb
698 + );
699 + }else if( p->errCode==SQLITE_OK ){
700 + pStmt = p->pGetPage;
701 + }
702 +
703 + if( pStmt ){
704 + sqlite3_bind_int64(pStmt, 1, pgno);
705 + if( SQLITE_ROW==sqlite3_step(pStmt) ){
706 + const u8 *aPg;
707 + int nPg;
708 + assert( p->errCode==SQLITE_OK );
709 + aPg = sqlite3_column_blob(pStmt, 0);
710 + nPg = sqlite3_column_bytes(pStmt, 0);
711 + if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){
712 + aPg = p->pPage1Disk;
713 + }
714 + sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT);
715 + }
716 + recoverReset(p, pStmt);
717 + }
718 + }
719 +
720 + if( p->errCode ){
721 + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1);
722 + sqlite3_result_error_code(pCtx, p->errCode);
723 + }
724 +}
725 +
726 +/*
727 +** Find a string that is not found anywhere in z[]. Return a pointer
728 +** to that string.
729 +**
730 +** Try to use zA and zB first. If both of those are already found in z[]
731 +** then make up some string and store it in the buffer zBuf.
732 +*/
733 +static const char *recoverUnusedString(
734 + const char *z, /* Result must not appear anywhere in z */
735 + const char *zA, const char *zB, /* Try these first */
736 + char *zBuf /* Space to store a generated string */
737 +){
738 + unsigned i = 0;
739 + if( strstr(z, zA)==0 ) return zA;
740 + if( strstr(z, zB)==0 ) return zB;
741 + do{
742 + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++);
743 + }while( strstr(z,zBuf)!=0 );
744 + return zBuf;
745 +}
746 +
747 +/*
748 +** Implementation of scalar SQL function "escape_crnl". The argument passed to
749 +** this function is the output of built-in function quote(). If the first
750 +** character of the input is "'", indicating that the value passed to quote()
751 +** was a text value, then this function searches the input for "\n" and "\r"
752 +** characters and adds a wrapper similar to the following:
753 +**
754 +** replace(replace(<input>, '\n', char(10), '\r', char(13));
755 +**
756 +** Or, if the first character of the input is not "'", then a copy of the input
757 +** is returned.
758 +*/
759 +static void recoverEscapeCrnl(
760 + sqlite3_context *context,
761 + int argc,
762 + sqlite3_value **argv
763 +){
764 + const char *zText = (const char*)sqlite3_value_text(argv[0]);
765 + (void)argc;
766 + if( zText && zText[0]=='\'' ){
767 + int nText = sqlite3_value_bytes(argv[0]);
768 + int i;
769 + char zBuf1[20];
770 + char zBuf2[20];
771 + const char *zNL = 0;
772 + const char *zCR = 0;
773 + int nCR = 0;
774 + int nNL = 0;
775 +
776 + for(i=0; zText[i]; i++){
777 + if( zNL==0 && zText[i]=='\n' ){
778 + zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1);
779 + nNL = (int)strlen(zNL);
780 + }
781 + if( zCR==0 && zText[i]=='\r' ){
782 + zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2);
783 + nCR = (int)strlen(zCR);
784 + }
785 + }
786 +
787 + if( zNL || zCR ){
788 + int iOut = 0;
789 + i64 nMax = (nNL > nCR) ? nNL : nCR;
790 + i64 nAlloc = nMax * nText + (nMax+64)*2;
791 + char *zOut = (char*)sqlite3_malloc64(nAlloc);
792 + if( zOut==0 ){
793 + sqlite3_result_error_nomem(context);
794 + return;
795 + }
796 +
797 + if( zNL && zCR ){
798 + memcpy(&zOut[iOut], "replace(replace(", 16);
799 + iOut += 16;
800 + }else{
801 + memcpy(&zOut[iOut], "replace(", 8);
802 + iOut += 8;
803 + }
804 + for(i=0; zText[i]; i++){
805 + if( zText[i]=='\n' ){
806 + memcpy(&zOut[iOut], zNL, nNL);
807 + iOut += nNL;
808 + }else if( zText[i]=='\r' ){
809 + memcpy(&zOut[iOut], zCR, nCR);
810 + iOut += nCR;
811 + }else{
812 + zOut[iOut] = zText[i];
813 + iOut++;
814 + }
815 + }
816 +
817 + if( zNL ){
818 + memcpy(&zOut[iOut], ",'", 2); iOut += 2;
819 + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL;
820 + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12;
821 + }
822 + if( zCR ){
823 + memcpy(&zOut[iOut], ",'", 2); iOut += 2;
824 + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR;
825 + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12;
826 + }
827 +
828 + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT);
829 + sqlite3_free(zOut);
830 + return;
831 + }
832 + }
833 +
834 + sqlite3_result_value(context, argv[0]);
835 +}
836 +
837 +/*
838 +** This function is a no-op if recover handle p already contains an error
839 +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in
840 +** this case.
841 +**
842 +** Otherwise, attempt to populate temporary table "recovery.schema" with the
843 +** parts of the database schema that can be extracted from the input database.
844 +**
845 +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
846 +** and error message are left in the recover handle and a copy of the
847 +** error code returned. It is not considered an error if part of all of
848 +** the database schema cannot be recovered due to corruption.
849 +*/
850 +static int recoverCacheSchema(sqlite3_recover *p){
851 + return recoverExec(p, p->dbOut,
852 + "WITH RECURSIVE pages(p) AS ("
853 + " SELECT 1"
854 + " UNION"
855 + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p"
856 + ")"
857 + "INSERT INTO recovery.schema SELECT"
858 + " max(CASE WHEN field=0 THEN value ELSE NULL END),"
859 + " max(CASE WHEN field=1 THEN value ELSE NULL END),"
860 + " max(CASE WHEN field=2 THEN value ELSE NULL END),"
861 + " max(CASE WHEN field=3 THEN value ELSE NULL END),"
862 + " max(CASE WHEN field=4 THEN value ELSE NULL END)"
863 + "FROM sqlite_dbdata('getpage()') WHERE pgno IN ("
864 + " SELECT p FROM pages"
865 + ") GROUP BY pgno, cell"
866 + );
867 +}
868 +
869 +/*
870 +** If this recover handle is not in SQL callback mode (i.e. was not created
871 +** using sqlite3_recover_init_sql()) of if an error has already occurred,
872 +** this function is a no-op. Otherwise, issue a callback with SQL statement
873 +** zSql as the parameter.
874 +**
875 +** If the callback returns non-zero, set the recover handle error code to
876 +** the value returned (so that the caller will abandon processing).
877 +*/
878 +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){
879 + if( p->errCode==SQLITE_OK && p->xSql ){
880 + int res = p->xSql(p->pSqlCtx, zSql);
881 + if( res ){
882 + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res);
883 + }
884 + }
885 +}
886 +
887 +/*
888 +** Transfer the following settings from the input database to the output
889 +** database:
890 +**
891 +** + page-size,
892 +** + auto-vacuum settings,
893 +** + database encoding,
894 +** + user-version (PRAGMA user_version), and
895 +** + application-id (PRAGMA application_id), and
896 +*/
897 +static void recoverTransferSettings(sqlite3_recover *p){
898 + const char *aPragma[] = {
899 + "encoding",
900 + "page_size",
901 + "auto_vacuum",
902 + "user_version",
903 + "application_id"
904 + };
905 + int ii;
906 +
907 + /* Truncate the output database to 0 pages in size. This is done by
908 + ** opening a new, empty, temp db, then using the backup API to clobber
909 + ** any existing output db with a copy of it. */
910 + if( p->errCode==SQLITE_OK ){
911 + sqlite3 *db2 = 0;
912 + int rc = sqlite3_open("", &db2);
913 + if( rc!=SQLITE_OK ){
914 + recoverDbError(p, db2);
915 + return;
916 + }
917 +
918 + for(ii=0; ii<(int)(sizeof(aPragma)/sizeof(aPragma[0])); ii++){
919 + const char *zPrag = aPragma[ii];
920 + sqlite3_stmt *p1 = 0;
921 + p1 = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.%s", p->zDb, zPrag);
922 + if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){
923 + const char *zArg = (const char*)sqlite3_column_text(p1, 0);
924 + char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg);
925 + recoverSqlCallback(p, z2);
926 + recoverExec(p, db2, z2);
927 + sqlite3_free(z2);
928 + if( zArg==0 ){
929 + recoverError(p, SQLITE_NOMEM, 0);
930 + }
931 + }
932 + recoverFinalize(p, p1);
933 + }
934 + recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;");
935 +
936 + if( p->errCode==SQLITE_OK ){
937 + sqlite3 *db = p->dbOut;
938 + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main");
939 + if( pBackup ){
940 + sqlite3_backup_step(pBackup, -1);
941 + p->errCode = sqlite3_backup_finish(pBackup);
942 + }else{
943 + recoverDbError(p, db);
944 + }
945 + }
946 +
947 + sqlite3_close(db2);
948 + }
949 +}
950 +
951 +/*
952 +** This function is a no-op if recover handle p already contains an error
953 +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in
954 +** this case.
955 +**
956 +** Otherwise, an attempt is made to open the output database, attach
957 +** and create the schema of the temporary database used to store
958 +** intermediate data, and to register all required user functions and
959 +** virtual table modules with the output handle.
960 +**
961 +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
962 +** and error message are left in the recover handle and a copy of the
963 +** error code returned.
964 +*/
965 +static int recoverOpenOutput(sqlite3_recover *p){
966 + struct Func {
967 + const char *zName;
968 + int nArg;
969 + void (*xFunc)(sqlite3_context*,int,sqlite3_value **);
970 + } aFunc[] = {
971 + { "getpage", 1, recoverGetPage },
972 + { "page_is_used", 1, recoverPageIsUsed },
973 + { "read_i32", 2, recoverReadI32 },
974 + { "escape_crnl", 1, recoverEscapeCrnl },
975 + };
976 +
977 + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE;
978 + sqlite3 *db = 0; /* New database handle */
979 + int ii; /* For iterating through aFunc[] */
980 +
981 + assert( p->dbOut==0 );
982 +
983 + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){
984 + recoverDbError(p, db);
985 + }
986 +
987 + /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules.
988 + ** These two are registered with the output database handle - this
989 + ** module depends on the input handle supporting the sqlite_dbpage
990 + ** virtual table only. */
991 + if( p->errCode==SQLITE_OK ){
992 + p->errCode = sqlite3_dbdata_init(db, 0, 0);
993 + }
994 +
995 + /* Register the custom user-functions with the output handle. */
996 + for(ii=0;
997 + p->errCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0]));
998 + ii++){
999 + p->errCode = sqlite3_create_function(db, aFunc[ii].zName,
1000 + aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0
1001 + );
1002 + }
1003 +
1004 + p->dbOut = db;
1005 + return p->errCode;
1006 +}
1007 +
1008 +/*
1009 +** Attach the auxiliary database 'recovery' to the output database handle.
1010 +** This temporary database is used during the recovery process and then
1011 +** discarded.
1012 +*/
1013 +static void recoverOpenRecovery(sqlite3_recover *p){
1014 + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb);
1015 + recoverExec(p, p->dbOut, zSql);
1016 + recoverExec(p, p->dbOut,
1017 + "PRAGMA writable_schema = 1;"
1018 + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);"
1019 + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);"
1020 + );
1021 + sqlite3_free(zSql);
1022 +}
1023 +
1024 +
1025 +/*
1026 +** This function is a no-op if recover handle p already contains an error
1027 +** (if p->errCode!=SQLITE_OK).
1028 +**
1029 +** Otherwise, argument zName must be the name of a table that has just been
1030 +** created in the output database. This function queries the output db
1031 +** for the schema of said table, and creates a RecoverTable object to
1032 +** store the schema in memory. The new RecoverTable object is linked into
1033 +** the list at sqlite3_recover.pTblList.
1034 +**
1035 +** Parameter iRoot must be the root page of table zName in the INPUT
1036 +** database.
1037 +*/
1038 +static void recoverAddTable(
1039 + sqlite3_recover *p,
1040 + const char *zName, /* Name of table created in output db */
1041 + i64 iRoot /* Root page of same table in INPUT db */
1042 +){
1043 + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut,
1044 + "PRAGMA table_xinfo(%Q)", zName
1045 + );
1046 +
1047 + if( pStmt ){
1048 + int iPk = -1;
1049 + int iBind = 1;
1050 + RecoverTable *pNew = 0;
1051 + int nCol = 0;
1052 + int nName = recoverStrlen(zName);
1053 + int nByte = 0;
1054 + while( sqlite3_step(pStmt)==SQLITE_ROW ){
1055 + nCol++;
1056 + nByte += (sqlite3_column_bytes(pStmt, 1)+1);
1057 + }
1058 + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1;
1059 + recoverReset(p, pStmt);
1060 +
1061 + pNew = recoverMalloc(p, nByte);
1062 + if( pNew ){
1063 + int i = 0;
1064 + int iField = 0;
1065 + char *csr = 0;
1066 + pNew->aCol = (RecoverColumn*)&pNew[1];
1067 + pNew->zTab = csr = (char*)&pNew->aCol[nCol];
1068 + pNew->nCol = nCol;
1069 + pNew->iRoot = iRoot;
1070 + memcpy(csr, zName, nName);
1071 + csr += nName+1;
1072 +
1073 + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){
1074 + int iPKF = sqlite3_column_int(pStmt, 5);
1075 + int n = sqlite3_column_bytes(pStmt, 1);
1076 + const char *z = (const char*)sqlite3_column_text(pStmt, 1);
1077 + const char *zType = (const char*)sqlite3_column_text(pStmt, 2);
1078 + int eHidden = sqlite3_column_int(pStmt, 6);
1079 +
1080 + if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i;
1081 + if( iPKF>1 ) iPk = -2;
1082 + pNew->aCol[i].zCol = csr;
1083 + pNew->aCol[i].eHidden = eHidden;
1084 + if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){
1085 + pNew->aCol[i].iField = -1;
1086 + }else{
1087 + pNew->aCol[i].iField = iField++;
1088 + }
1089 + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL
1090 + && eHidden!=RECOVER_EHIDDEN_STORED
1091 + ){
1092 + pNew->aCol[i].iBind = iBind++;
1093 + }
1094 + memcpy(csr, z, n);
1095 + csr += (n+1);
1096 + }
1097 +
1098 + pNew->pNext = p->pTblList;
1099 + p->pTblList = pNew;
1100 + pNew->bIntkey = 1;
1101 + }
1102 +
1103 + recoverFinalize(p, pStmt);
1104 +
1105 + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName);
1106 + while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
1107 + int iField = sqlite3_column_int(pStmt, 0);
1108 + int iCol = sqlite3_column_int(pStmt, 1);
1109 +
1110 + assert( iCol<pNew->nCol );
1111 + pNew->aCol[iCol].iField = iField;
1112 +
1113 + pNew->bIntkey = 0;
1114 + iPk = -2;
1115 + }
1116 + recoverFinalize(p, pStmt);
1117 +
1118 + if( p->errCode==SQLITE_OK ){
1119 + if( iPk>=0 ){
1120 + pNew->aCol[iPk].bIPK = 1;
1121 + }else if( pNew->bIntkey ){
1122 + pNew->iRowidBind = iBind++;
1123 + }
1124 + }
1125 + }
1126 +}
1127 +
1128 +/*
1129 +** This function is called after recoverCacheSchema() has cached those parts
1130 +** of the input database schema that could be recovered in temporary table
1131 +** "recovery.schema". This function creates in the output database copies
1132 +** of all parts of that schema that must be created before the tables can
1133 +** be populated. Specifically, this means:
1134 +**
1135 +** * all tables that are not VIRTUAL, and
1136 +** * UNIQUE indexes.
1137 +**
1138 +** If the recovery handle uses SQL callbacks, then callbacks containing
1139 +** the associated "CREATE TABLE" and "CREATE INDEX" statements are made.
1140 +**
1141 +** Additionally, records are added to the sqlite_schema table of the
1142 +** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE
1143 +** records are written directly to sqlite_schema, not actually executed.
1144 +** If the handle is in SQL callback mode, then callbacks are invoked
1145 +** with equivalent SQL statements.
1146 +*/
1147 +static int recoverWriteSchema1(sqlite3_recover *p){
1148 + sqlite3_stmt *pSelect = 0;
1149 + sqlite3_stmt *pTblname = 0;
1150 +
1151 + pSelect = recoverPrepare(p, p->dbOut,
1152 + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS ("
1153 + " SELECT rootpage, name, sql, "
1154 + " type='table', "
1155 + " sql LIKE 'create virtual%',"
1156 + " (type='index' AND (sql LIKE '%unique%' OR ?1))"
1157 + " FROM recovery.schema"
1158 + ")"
1159 + "SELECT rootpage, tbl, isVirtual, name, sql"
1160 + " FROM dbschema "
1161 + " WHERE tbl OR isIndex"
1162 + " ORDER BY tbl DESC, name=='sqlite_sequence' DESC"
1163 + );
1164 +
1165 + pTblname = recoverPrepare(p, p->dbOut,
1166 + "SELECT name FROM sqlite_schema "
1167 + "WHERE type='table' ORDER BY rowid DESC LIMIT 1"
1168 + );
1169 +
1170 + if( pSelect ){
1171 + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes);
1172 + while( sqlite3_step(pSelect)==SQLITE_ROW ){
1173 + i64 iRoot = sqlite3_column_int64(pSelect, 0);
1174 + int bTable = sqlite3_column_int(pSelect, 1);
1175 + int bVirtual = sqlite3_column_int(pSelect, 2);
1176 + const char *zName = (const char*)sqlite3_column_text(pSelect, 3);
1177 + const char *zSql = (const char*)sqlite3_column_text(pSelect, 4);
1178 + char *zFree = 0;
1179 + int rc = SQLITE_OK;
1180 +
1181 + if( bVirtual ){
1182 + zSql = (const char*)(zFree = recoverMPrintf(p,
1183 + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)",
1184 + zName, zName, zSql
1185 + ));
1186 + }
1187 + rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0);
1188 + if( rc==SQLITE_OK ){
1189 + recoverSqlCallback(p, zSql);
1190 + if( bTable && !bVirtual ){
1191 + if( SQLITE_ROW==sqlite3_step(pTblname) ){
1192 + const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0);
1193 + recoverAddTable(p, zTbl, iRoot);
1194 + }
1195 + recoverReset(p, pTblname);
1196 + }
1197 + }else if( rc!=SQLITE_ERROR ){
1198 + recoverDbError(p, p->dbOut);
1199 + }
1200 + sqlite3_free(zFree);
1201 + }
1202 + }
1203 + recoverFinalize(p, pSelect);
1204 + recoverFinalize(p, pTblname);
1205 +
1206 + return p->errCode;
1207 +}
1208 +
1209 +/*
1210 +** This function is called after the output database has been populated. It
1211 +** adds all recovered schema elements that were not created in the output
1212 +** database by recoverWriteSchema1() - everything except for tables and
1213 +** UNIQUE indexes. Specifically:
1214 +**
1215 +** * views,
1216 +** * triggers,
1217 +** * non-UNIQUE indexes.
1218 +**
1219 +** If the recover handle is in SQL callback mode, then equivalent callbacks
1220 +** are issued to create the schema elements.
1221 +*/
1222 +static int recoverWriteSchema2(sqlite3_recover *p){
1223 + sqlite3_stmt *pSelect = 0;
1224 +
1225 + pSelect = recoverPrepare(p, p->dbOut,
1226 + p->bSlowIndexes ?
1227 + "SELECT rootpage, sql FROM recovery.schema "
1228 + " WHERE type!='table' AND type!='index'"
1229 + :
1230 + "SELECT rootpage, sql FROM recovery.schema "
1231 + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')"
1232 + );
1233 +
1234 + if( pSelect ){
1235 + while( sqlite3_step(pSelect)==SQLITE_ROW ){
1236 + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
1237 + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0);
1238 + if( rc==SQLITE_OK ){
1239 + recoverSqlCallback(p, zSql);
1240 + }else if( rc!=SQLITE_ERROR ){
1241 + recoverDbError(p, p->dbOut);
1242 + }
1243 + }
1244 + }
1245 + recoverFinalize(p, pSelect);
1246 +
1247 + return p->errCode;
1248 +}
1249 +
1250 +/*
1251 +** This function is a no-op if recover handle p already contains an error
1252 +** (if p->errCode!=SQLITE_OK). In this case it returns NULL.
1253 +**
1254 +** Otherwise, if the recover handle is configured to create an output
1255 +** database (was created by sqlite3_recover_init()), then this function
1256 +** prepares and returns an SQL statement to INSERT a new record into table
1257 +** pTab, assuming the first nField fields of a record extracted from disk
1258 +** are valid.
1259 +**
1260 +** For example, if table pTab is:
1261 +**
1262 +** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e);
1263 +**
1264 +** And nField is 4, then the SQL statement prepared and returned is:
1265 +**
1266 +** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3);
1267 +**
1268 +** In this case even though 4 values were extracted from the input db,
1269 +** only 3 are written to the output, as the generated STORED column
1270 +** cannot be written.
1271 +**
1272 +** If the recover handle is in SQL callback mode, then the SQL statement
1273 +** prepared is such that evaluating it returns a single row containing
1274 +** a single text value - itself an SQL statement similar to the above,
1275 +** except with SQL literals in place of the variables. For example:
1276 +**
1277 +** SELECT 'INSERT INTO (a, c, d) VALUES ('
1278 +** || quote(?1) || ', '
1279 +** || quote(?2) || ', '
1280 +** || quote(?3) || ')';
1281 +**
1282 +** In either case, it is the responsibility of the caller to eventually
1283 +** free the statement handle using sqlite3_finalize().
1284 +*/
1285 +static sqlite3_stmt *recoverInsertStmt(
1286 + sqlite3_recover *p,
1287 + RecoverTable *pTab,
1288 + int nField
1289 +){
1290 + sqlite3_stmt *pRet = 0;
1291 + const char *zSep = "";
1292 + const char *zSqlSep = "";
1293 + char *zSql = 0;
1294 + char *zFinal = 0;
1295 + char *zBind = 0;
1296 + int ii;
1297 + int bSql = p->xSql ? 1 : 0;
1298 +
1299 + if( nField<=0 ) return 0;
1300 +
1301 + assert( nField<=pTab->nCol );
1302 +
1303 + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab);
1304 +
1305 + if( pTab->iRowidBind ){
1306 + assert( pTab->bIntkey );
1307 + zSql = recoverMPrintf(p, "%z_rowid_", zSql);
1308 + if( bSql ){
1309 + zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind);
1310 + }else{
1311 + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind);
1312 + }
1313 + zSqlSep = "||', '||";
1314 + zSep = ", ";
1315 + }
1316 +
1317 + for(ii=0; ii<nField; ii++){
1318 + int eHidden = pTab->aCol[ii].eHidden;
1319 + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL
1320 + && eHidden!=RECOVER_EHIDDEN_STORED
1321 + ){
1322 + assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 );
1323 + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol);
1324 +
1325 + if( bSql ){
1326 + zBind = recoverMPrintf(p,
1327 + "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind
1328 + );
1329 + zSqlSep = "||', '||";
1330 + }else{
1331 + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind);
1332 + }
1333 + zSep = ", ";
1334 + }
1335 + }
1336 +
1337 + if( bSql ){
1338 + zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'",
1339 + zSql, zBind
1340 + );
1341 + }else{
1342 + zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind);
1343 + }
1344 +
1345 + pRet = recoverPrepare(p, p->dbOut, zFinal);
1346 + sqlite3_free(zSql);
1347 + sqlite3_free(zBind);
1348 + sqlite3_free(zFinal);
1349 +
1350 + return pRet;
1351 +}
1352 +
1353 +
1354 +/*
1355 +** Search the list of RecoverTable objects at p->pTblList for one that
1356 +** has root page iRoot in the input database. If such an object is found,
1357 +** return a pointer to it. Otherwise, return NULL.
1358 +*/
1359 +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){
1360 + RecoverTable *pRet = 0;
1361 + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext);
1362 + return pRet;
1363 +}
1364 +
1365 +/*
1366 +** This function attempts to create a lost and found table within the
1367 +** output db. If successful, it returns a pointer to a buffer containing
1368 +** the name of the new table. It is the responsibility of the caller to
1369 +** eventually free this buffer using sqlite3_free().
1370 +**
1371 +** If an error occurs, NULL is returned and an error code and error
1372 +** message left in the recover handle.
1373 +*/
1374 +static char *recoverLostAndFoundCreate(
1375 + sqlite3_recover *p, /* Recover object */
1376 + int nField /* Number of column fields in new table */
1377 +){
1378 + char *zTbl = 0;
1379 + sqlite3_stmt *pProbe = 0;
1380 + int ii = 0;
1381 +
1382 + pProbe = recoverPrepare(p, p->dbOut,
1383 + "SELECT 1 FROM sqlite_schema WHERE name=?"
1384 + );
1385 + for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){
1386 + int bFail = 0;
1387 + if( ii<0 ){
1388 + zTbl = recoverMPrintf(p, "%s", p->zLostAndFound);
1389 + }else{
1390 + zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii);
1391 + }
1392 +
1393 + if( p->errCode==SQLITE_OK ){
1394 + sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC);
1395 + if( SQLITE_ROW==sqlite3_step(pProbe) ){
1396 + bFail = 1;
1397 + }
1398 + recoverReset(p, pProbe);
1399 + }
1400 +
1401 + if( bFail ){
1402 + sqlite3_clear_bindings(pProbe);
1403 + sqlite3_free(zTbl);
1404 + zTbl = 0;
1405 + }
1406 + }
1407 + recoverFinalize(p, pProbe);
1408 +
1409 + if( zTbl ){
1410 + const char *zSep = 0;
1411 + char *zField = 0;
1412 + char *zSql = 0;
1413 +
1414 + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, ";
1415 + for(ii=0; p->errCode==SQLITE_OK && ii<nField; ii++){
1416 + zField = recoverMPrintf(p, "%z%sc%d", zField, zSep, ii);
1417 + zSep = ", ";
1418 + }
1419 +
1420 + zSql = recoverMPrintf(p, "CREATE TABLE %s(%s)", zTbl, zField);
1421 + sqlite3_free(zField);
1422 +
1423 + recoverExec(p, p->dbOut, zSql);
1424 + recoverSqlCallback(p, zSql);
1425 + sqlite3_free(zSql);
1426 + }else if( p->errCode==SQLITE_OK ){
1427 + recoverError(
1428 + p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound
1429 + );
1430 + }
1431 +
1432 + return zTbl;
1433 +}
1434 +
1435 +/*
1436 +** Synthesize and prepare an INSERT statement to write to the lost_and_found
1437 +** table in the output database. The name of the table is zTab, and it has
1438 +** nField c* fields.
1439 +*/
1440 +static sqlite3_stmt *recoverLostAndFoundInsert(
1441 + sqlite3_recover *p,
1442 + const char *zTab,
1443 + int nField
1444 +){
1445 + int nTotal = nField + 4;
1446 + int ii;
1447 + char *zBind = 0;
1448 + sqlite3_stmt *pRet = 0;
1449 +
1450 + if( p->xSql==0 ){
1451 + for(ii=0; ii<nTotal; ii++){
1452 + zBind = recoverMPrintf(p, "%z%s?", zBind, zBind?", ":"", ii);
1453 + }
1454 + pRet = recoverPreparePrintf(
1455 + p, p->dbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind
1456 + );
1457 + }else{
1458 + const char *zSep = "";
1459 + for(ii=0; ii<nTotal; ii++){
1460 + zBind = recoverMPrintf(p, "%z%squote(?)", zBind, zSep);
1461 + zSep = "|| ', ' ||";
1462 + }
1463 + pRet = recoverPreparePrintf(
1464 + p, p->dbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind
1465 + );
1466 + }
1467 +
1468 + sqlite3_free(zBind);
1469 + return pRet;
1470 +}
1471 +
1472 +/*
1473 +** Input database page iPg contains data that will be written to the
1474 +** lost-and-found table of the output database. This function attempts
1475 +** to identify the root page of the tree that page iPg belonged to.
1476 +** If successful, it sets output variable (*piRoot) to the page number
1477 +** of the root page and returns SQLITE_OK. Otherwise, if an error occurs,
1478 +** an SQLite error code is returned and the final value of *piRoot
1479 +** undefined.
1480 +*/
1481 +static int recoverLostAndFoundFindRoot(
1482 + sqlite3_recover *p,
1483 + i64 iPg,
1484 + i64 *piRoot
1485 +){
1486 + RecoverStateLAF *pLaf = &p->laf;
1487 +
1488 + if( pLaf->pFindRoot==0 ){
1489 + pLaf->pFindRoot = recoverPrepare(p, p->dbOut,
1490 + "WITH RECURSIVE p(pgno) AS ("
1491 + " SELECT ?"
1492 + " UNION"
1493 + " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno"
1494 + ") "
1495 + "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno "
1496 + " AND m.parent IS NULL"
1497 + );
1498 + }
1499 + if( p->errCode==SQLITE_OK ){
1500 + sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg);
1501 + if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){
1502 + *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0);
1503 + }else{
1504 + *piRoot = iPg;
1505 + }
1506 + recoverReset(p, pLaf->pFindRoot);
1507 + }
1508 + return p->errCode;
1509 +}
1510 +
1511 +/*
1512 +** Recover data from page iPage of the input database and write it to
1513 +** the lost-and-found table in the output database.
1514 +*/
1515 +static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){
1516 + RecoverStateLAF *pLaf = &p->laf;
1517 + sqlite3_value **apVal = pLaf->apVal;
1518 + sqlite3_stmt *pPageData = pLaf->pPageData;
1519 + sqlite3_stmt *pInsert = pLaf->pInsert;
1520 +
1521 + int nVal = -1;
1522 + int iPrevCell = 0;
1523 + i64 iRoot = 0;
1524 + int bHaveRowid = 0;
1525 + i64 iRowid = 0;
1526 + int ii = 0;
1527 +
1528 + if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return;
1529 + sqlite3_bind_int64(pPageData, 1, iPage);
1530 + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){
1531 + int iCell = sqlite3_column_int64(pPageData, 0);
1532 + int iField = sqlite3_column_int64(pPageData, 1);
1533 +
1534 + if( iPrevCell!=iCell && nVal>=0 ){
1535 + /* Insert the new row */
1536 + sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */
1537 + sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */
1538 + sqlite3_bind_int(pInsert, 3, nVal); /* nfield */
1539 + if( bHaveRowid ){
1540 + sqlite3_bind_int64(pInsert, 4, iRowid); /* id */
1541 + }
1542 + for(ii=0; ii<nVal; ii++){
1543 + recoverBindValue(p, pInsert, 5+ii, apVal[ii]);
1544 + }
1545 + if( sqlite3_step(pInsert)==SQLITE_ROW ){
1546 + recoverSqlCallback(p, (const char*)sqlite3_column_text(pInsert, 0));
1547 + }
1548 + recoverReset(p, pInsert);
1549 +
1550 + /* Discard the accumulated row data */
1551 + for(ii=0; ii<nVal; ii++){
1552 + sqlite3_value_free(apVal[ii]);
1553 + apVal[ii] = 0;
1554 + }
1555 + sqlite3_clear_bindings(pInsert);
1556 + bHaveRowid = 0;
1557 + nVal = -1;
1558 + }
1559 +
1560 + if( iCell<0 ) break;
1561 +
1562 + if( iField<0 ){
1563 + assert( nVal==-1 );
1564 + iRowid = sqlite3_column_int64(pPageData, 2);
1565 + bHaveRowid = 1;
1566 + nVal = 0;
1567 + }else if( iField<pLaf->nMaxField ){
1568 + sqlite3_value *pVal = sqlite3_column_value(pPageData, 2);
1569 + apVal[iField] = sqlite3_value_dup(pVal);
1570 + assert( iField==nVal || (nVal==-1 && iField==0) );
1571 + nVal = iField+1;
1572 + if( apVal[iField]==0 ){
1573 + recoverError(p, SQLITE_NOMEM, 0);
1574 + }
1575 + }
1576 +
1577 + iPrevCell = iCell;
1578 + }
1579 + recoverReset(p, pPageData);
1580 +
1581 + for(ii=0; ii<nVal; ii++){
1582 + sqlite3_value_free(apVal[ii]);
1583 + apVal[ii] = 0;
1584 + }
1585 +}
1586 +
1587 +/*
1588 +** Perform one step (sqlite3_recover_step()) of work for the connection
1589 +** passed as the only argument, which is guaranteed to be in
1590 +** RECOVER_STATE_LOSTANDFOUND3 state - during which the lost-and-found
1591 +** table of the output database is populated with recovered data that can
1592 +** not be assigned to any recovered schema object.
1593 +*/
1594 +static int recoverLostAndFound3Step(sqlite3_recover *p){
1595 + RecoverStateLAF *pLaf = &p->laf;
1596 + if( p->errCode==SQLITE_OK ){
1597 + if( pLaf->pInsert==0 ){
1598 + return SQLITE_DONE;
1599 + }else{
1600 + if( p->errCode==SQLITE_OK ){
1601 + int res = sqlite3_step(pLaf->pAllPage);
1602 + if( res==SQLITE_ROW ){
1603 + i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0);
1604 + if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){
1605 + recoverLostAndFoundOnePage(p, iPage);
1606 + }
1607 + }else{
1608 + recoverReset(p, pLaf->pAllPage);
1609 + return SQLITE_DONE;
1610 + }
1611 + }
1612 + }
1613 + }
1614 + return SQLITE_OK;
1615 +}
1616 +
1617 +/*
1618 +** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3
1619 +** state - during which the lost-and-found table of the output database
1620 +** is populated with recovered data that can not be assigned to any
1621 +** recovered schema object.
1622 +*/
1623 +static void recoverLostAndFound3Init(sqlite3_recover *p){
1624 + RecoverStateLAF *pLaf = &p->laf;
1625 +
1626 + if( pLaf->nMaxField>0 ){
1627 + char *zTab = 0; /* Name of lost_and_found table */
1628 +
1629 + zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField);
1630 + pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField);
1631 + sqlite3_free(zTab);
1632 +
1633 + pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut,
1634 + "WITH RECURSIVE seq(ii) AS ("
1635 + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld"
1636 + ")"
1637 + "SELECT ii FROM seq" , p->laf.nPg
1638 + );
1639 + pLaf->pPageData = recoverPrepare(p, p->dbOut,
1640 + "SELECT cell, field, value "
1641 + "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? "
1642 + "UNION ALL "
1643 + "SELECT -1, -1, -1"
1644 + );
1645 +
1646 + pLaf->apVal = (sqlite3_value**)recoverMalloc(p,
1647 + pLaf->nMaxField*sizeof(sqlite3_value*)
1648 + );
1649 + }
1650 +}
1651 +
1652 +/*
1653 +** Initialize resources required in RECOVER_STATE_WRITING state - during which
1654 +** tables recovered from the schema of the input database are populated with
1655 +** recovered data.
1656 +*/
1657 +static int recoverWriteDataInit(sqlite3_recover *p){
1658 + RecoverStateW1 *p1 = &p->w1;
1659 + RecoverTable *pTbl = 0;
1660 + int nByte = 0;
1661 +
1662 + /* Figure out the maximum number of columns for any table in the schema */
1663 + assert( p1->nMax==0 );
1664 + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){
1665 + if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol;
1666 + }
1667 +
1668 + /* Allocate an array of (sqlite3_value*) in which to accumulate the values
1669 + ** that will be written to the output database in a single row. */
1670 + nByte = sizeof(sqlite3_value*) * (p1->nMax+1);
1671 + p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte);
1672 + if( p1->apVal==0 ) return p->errCode;
1673 +
1674 + /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT
1675 + ** to loop through cells that appear to belong to a single table (pSel). */
1676 + p1->pTbls = recoverPrepare(p, p->dbOut,
1677 + "SELECT rootpage FROM recovery.schema "
1678 + " WHERE type='table' AND (sql NOT LIKE 'create virtual%')"
1679 + " ORDER BY (tbl_name='sqlite_sequence') ASC"
1680 + );
1681 + p1->pSel = recoverPrepare(p, p->dbOut,
1682 + "WITH RECURSIVE pages(page) AS ("
1683 + " SELECT ?1"
1684 + " UNION"
1685 + " SELECT child FROM sqlite_dbptr('getpage()'), pages "
1686 + " WHERE pgno=page"
1687 + ") "
1688 + "SELECT page, cell, field, value "
1689 + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno "
1690 + "UNION ALL "
1691 + "SELECT 0, 0, 0, 0"
1692 + );
1693 +
1694 + return p->errCode;
1695 +}
1696 +
1697 +/*
1698 +** Clean up resources allocated by recoverWriteDataInit() (stuff in
1699 +** sqlite3_recover.w1).
1700 +*/
1701 +static void recoverWriteDataCleanup(sqlite3_recover *p){
1702 + RecoverStateW1 *p1 = &p->w1;
1703 + int ii;
1704 + for(ii=0; ii<p1->nVal; ii++){
1705 + sqlite3_value_free(p1->apVal[ii]);
1706 + }
1707 + sqlite3_free(p1->apVal);
1708 + recoverFinalize(p, p1->pInsert);
1709 + recoverFinalize(p, p1->pTbls);
1710 + recoverFinalize(p, p1->pSel);
1711 + memset(p1, 0, sizeof(*p1));
1712 +}
1713 +
1714 +/*
1715 +** Perform one step (sqlite3_recover_step()) of work for the connection
1716 +** passed as the only argument, which is guaranteed to be in
1717 +** RECOVER_STATE_WRITING state - during which tables recovered from the
1718 +** schema of the input database are populated with recovered data.
1719 +*/
1720 +static int recoverWriteDataStep(sqlite3_recover *p){
1721 + RecoverStateW1 *p1 = &p->w1;
1722 + sqlite3_stmt *pSel = p1->pSel;
1723 + sqlite3_value **apVal = p1->apVal;
1724 +
1725 + if( p->errCode==SQLITE_OK && p1->pTab==0 ){
1726 + if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){
1727 + i64 iRoot = sqlite3_column_int64(p1->pTbls, 0);
1728 + p1->pTab = recoverFindTable(p, iRoot);
1729 +
1730 + recoverFinalize(p, p1->pInsert);
1731 + p1->pInsert = 0;
1732 +
1733 + /* If this table is unknown, return early. The caller will invoke this
1734 + ** function again and it will move on to the next table. */
1735 + if( p1->pTab==0 ) return p->errCode;
1736 +
1737 + /* If this is the sqlite_sequence table, delete any rows added by
1738 + ** earlier INSERT statements on tables with AUTOINCREMENT primary
1739 + ** keys before recovering its contents. The p1->pTbls SELECT statement
1740 + ** is rigged to deliver "sqlite_sequence" last of all, so we don't
1741 + ** worry about it being modified after it is recovered. */
1742 + if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){
1743 + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence");
1744 + recoverSqlCallback(p, "DELETE FROM sqlite_sequence");
1745 + }
1746 +
1747 + /* Bind the root page of this table within the original database to
1748 + ** SELECT statement p1->pSel. The SELECT statement will then iterate
1749 + ** through cells that look like they belong to table pTab. */
1750 + sqlite3_bind_int64(pSel, 1, iRoot);
1751 +
1752 + p1->nVal = 0;
1753 + p1->bHaveRowid = 0;
1754 + p1->iPrevPage = -1;
1755 + p1->iPrevCell = -1;
1756 + }else{
1757 + return SQLITE_DONE;
1758 + }
1759 + }
1760 + assert( p->errCode!=SQLITE_OK || p1->pTab );
1761 +
1762 + if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){
1763 + RecoverTable *pTab = p1->pTab;
1764 +
1765 + i64 iPage = sqlite3_column_int64(pSel, 0);
1766 + int iCell = sqlite3_column_int(pSel, 1);
1767 + int iField = sqlite3_column_int(pSel, 2);
1768 + sqlite3_value *pVal = sqlite3_column_value(pSel, 3);
1769 + int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell);
1770 +
1771 + assert( bNewCell==0 || (iField==-1 || iField==0) );
1772 + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol );
1773 +
1774 + if( bNewCell ){
1775 + int ii = 0;
1776 + if( p1->nVal>=0 ){
1777 + if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){
1778 + recoverFinalize(p, p1->pInsert);
1779 + p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal);
1780 + p1->nInsert = p1->nVal;
1781 + }
1782 + if( p1->nVal>0 ){
1783 + sqlite3_stmt *pInsert = p1->pInsert;
1784 + for(ii=0; ii<pTab->nCol; ii++){
1785 + RecoverColumn *pCol = &pTab->aCol[ii];
1786 + int iBind = pCol->iBind;
1787 + if( iBind>0 ){
1788 + if( pCol->bIPK ){
1789 + sqlite3_bind_int64(pInsert, iBind, p1->iRowid);
1790 + }else if( pCol->iField<p1->nVal ){
1791 + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]);
1792 + }
1793 + }
1794 + }
1795 + if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){
1796 + sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid);
1797 + }
1798 + if( SQLITE_ROW==sqlite3_step(pInsert) ){
1799 + const char *z = (const char*)sqlite3_column_text(pInsert, 0);
1800 + recoverSqlCallback(p, z);
1801 + }
1802 + recoverReset(p, pInsert);
1803 + assert( p->errCode || pInsert );
1804 + if( pInsert ) sqlite3_clear_bindings(pInsert);
1805 + }
1806 + }
1807 +
1808 + for(ii=0; ii<p1->nVal; ii++){
1809 + sqlite3_value_free(apVal[ii]);
1810 + apVal[ii] = 0;
1811 + }
1812 + p1->nVal = -1;
1813 + p1->bHaveRowid = 0;
1814 + }
1815 +
1816 + if( iPage!=0 ){
1817 + if( iField<0 ){
1818 + p1->iRowid = sqlite3_column_int64(pSel, 3);
1819 + assert( p1->nVal==-1 );
1820 + p1->nVal = 0;
1821 + p1->bHaveRowid = 1;
1822 + }else if( iField<pTab->nCol ){
1823 + assert( apVal[iField]==0 );
1824 + apVal[iField] = sqlite3_value_dup( pVal );
1825 + if( apVal[iField]==0 ){
1826 + recoverError(p, SQLITE_NOMEM, 0);
1827 + }
1828 + p1->nVal = iField+1;
1829 + }
1830 + p1->iPrevCell = iCell;
1831 + p1->iPrevPage = iPage;
1832 + }
1833 + }else{
1834 + recoverReset(p, pSel);
1835 + p1->pTab = 0;
1836 + }
1837 +
1838 + return p->errCode;
1839 +}
1840 +
1841 +/*
1842 +** Initialize resources required by sqlite3_recover_step() in
1843 +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not
1844 +** already allocated to a recovered schema element is determined.
1845 +*/
1846 +static void recoverLostAndFound1Init(sqlite3_recover *p){
1847 + RecoverStateLAF *pLaf = &p->laf;
1848 + sqlite3_stmt *pStmt = 0;
1849 +
1850 + assert( p->laf.pUsed==0 );
1851 + pLaf->nPg = recoverPageCount(p);
1852 + pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg);
1853 +
1854 + /* Prepare a statement to iterate through all pages that are part of any tree
1855 + ** in the recoverable part of the input database schema to the bitmap. And,
1856 + ** if !p->bFreelistCorrupt, add all pages that appear to be part of the
1857 + ** freelist. */
1858 + pStmt = recoverPrepare(
1859 + p, p->dbOut,
1860 + "WITH trunk(pgno) AS ("
1861 + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0"
1862 + " UNION"
1863 + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0"
1864 + "),"
1865 + "trunkdata(pgno, data) AS ("
1866 + " SELECT pgno, getpage(pgno) FROM trunk"
1867 + "),"
1868 + "freelist(data, n, freepgno) AS ("
1869 + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata"
1870 + " UNION ALL"
1871 + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0"
1872 + "),"
1873 + ""
1874 + "roots(r) AS ("
1875 + " SELECT 1 UNION ALL"
1876 + " SELECT rootpage FROM recovery.schema WHERE rootpage>0"
1877 + "),"
1878 + "used(page) AS ("
1879 + " SELECT r FROM roots"
1880 + " UNION"
1881 + " SELECT child FROM sqlite_dbptr('getpage()'), used "
1882 + " WHERE pgno=page"
1883 + ") "
1884 + "SELECT page FROM used"
1885 + " UNION ALL "
1886 + "SELECT freepgno FROM freelist WHERE NOT ?"
1887 + );
1888 + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt);
1889 + pLaf->pUsedPages = pStmt;
1890 +}
1891 +
1892 +/*
1893 +** Perform one step (sqlite3_recover_step()) of work for the connection
1894 +** passed as the only argument, which is guaranteed to be in
1895 +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not
1896 +** already allocated to a recovered schema element is determined.
1897 +*/
1898 +static int recoverLostAndFound1Step(sqlite3_recover *p){
1899 + RecoverStateLAF *pLaf = &p->laf;
1900 + int rc = p->errCode;
1901 + if( rc==SQLITE_OK ){
1902 + rc = sqlite3_step(pLaf->pUsedPages);
1903 + if( rc==SQLITE_ROW ){
1904 + i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0);
1905 + recoverBitmapSet(pLaf->pUsed, iPg);
1906 + rc = SQLITE_OK;
1907 + }else{
1908 + recoverFinalize(p, pLaf->pUsedPages);
1909 + pLaf->pUsedPages = 0;
1910 + }
1911 + }
1912 + return rc;
1913 +}
1914 +
1915 +/*
1916 +** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2
1917 +** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1
1918 +** are sorted into sets that likely belonged to the same database tree.
1919 +*/
1920 +static void recoverLostAndFound2Init(sqlite3_recover *p){
1921 + RecoverStateLAF *pLaf = &p->laf;
1922 +
1923 + assert( p->laf.pAllAndParent==0 );
1924 + assert( p->laf.pMapInsert==0 );
1925 + assert( p->laf.pMaxField==0 );
1926 + assert( p->laf.nMaxField==0 );
1927 +
1928 + pLaf->pMapInsert = recoverPrepare(p, p->dbOut,
1929 + "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)"
1930 + );
1931 + pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut,
1932 + "WITH RECURSIVE seq(ii) AS ("
1933 + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld"
1934 + ")"
1935 + "SELECT pgno, child FROM sqlite_dbptr('getpage()') "
1936 + " UNION ALL "
1937 + "SELECT NULL, ii FROM seq", p->laf.nPg
1938 + );
1939 + pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut,
1940 + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?"
1941 + );
1942 +}
1943 +
1944 +/*
1945 +** Perform one step (sqlite3_recover_step()) of work for the connection
1946 +** passed as the only argument, which is guaranteed to be in
1947 +** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified
1948 +** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged
1949 +** to the same database tree.
1950 +*/
1951 +static int recoverLostAndFound2Step(sqlite3_recover *p){
1952 + RecoverStateLAF *pLaf = &p->laf;
1953 + if( p->errCode==SQLITE_OK ){
1954 + int res = sqlite3_step(pLaf->pAllAndParent);
1955 + if( res==SQLITE_ROW ){
1956 + i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1);
1957 + if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){
1958 + sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild);
1959 + sqlite3_bind_value(pLaf->pMapInsert, 2,
1960 + sqlite3_column_value(pLaf->pAllAndParent, 0)
1961 + );
1962 + sqlite3_step(pLaf->pMapInsert);
1963 + recoverReset(p, pLaf->pMapInsert);
1964 + sqlite3_bind_int64(pLaf->pMaxField, 1, iChild);
1965 + if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){
1966 + int nMax = sqlite3_column_int(pLaf->pMaxField, 0);
1967 + if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax;
1968 + }
1969 + recoverReset(p, pLaf->pMaxField);
1970 + }
1971 + }else{
1972 + recoverFinalize(p, pLaf->pAllAndParent);
1973 + pLaf->pAllAndParent =0;
1974 + return SQLITE_DONE;
1975 + }
1976 + }
1977 + return p->errCode;
1978 +}
1979 +
1980 +/*
1981 +** Free all resources allocated as part of sqlite3_recover_step() calls
1982 +** in one of the RECOVER_STATE_LOSTANDFOUND[123] states.
1983 +*/
1984 +static void recoverLostAndFoundCleanup(sqlite3_recover *p){
1985 + recoverBitmapFree(p->laf.pUsed);
1986 + p->laf.pUsed = 0;
1987 + sqlite3_finalize(p->laf.pUsedPages);
1988 + sqlite3_finalize(p->laf.pAllAndParent);
1989 + sqlite3_finalize(p->laf.pMapInsert);
1990 + sqlite3_finalize(p->laf.pMaxField);
1991 + sqlite3_finalize(p->laf.pFindRoot);
1992 + sqlite3_finalize(p->laf.pInsert);
1993 + sqlite3_finalize(p->laf.pAllPage);
1994 + sqlite3_finalize(p->laf.pPageData);
1995 + p->laf.pUsedPages = 0;
1996 + p->laf.pAllAndParent = 0;
1997 + p->laf.pMapInsert = 0;
1998 + p->laf.pMaxField = 0;
1999 + p->laf.pFindRoot = 0;
2000 + p->laf.pInsert = 0;
2001 + p->laf.pAllPage = 0;
2002 + p->laf.pPageData = 0;
2003 + sqlite3_free(p->laf.apVal);
2004 + p->laf.apVal = 0;
2005 +}
2006 +
2007 +/*
2008 +** Free all resources allocated as part of sqlite3_recover_step() calls.
2009 +*/
2010 +static void recoverFinalCleanup(sqlite3_recover *p){
2011 + RecoverTable *pTab = 0;
2012 + RecoverTable *pNext = 0;
2013 +
2014 + recoverWriteDataCleanup(p);
2015 + recoverLostAndFoundCleanup(p);
2016 +
2017 + for(pTab=p->pTblList; pTab; pTab=pNext){
2018 + pNext = pTab->pNext;
2019 + sqlite3_free(pTab);
2020 + }
2021 + p->pTblList = 0;
2022 + sqlite3_finalize(p->pGetPage);
2023 + p->pGetPage = 0;
2024 + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0);
2025 +
2026 + {
2027 +#ifndef NDEBUG
2028 + int res =
2029 +#endif
2030 + sqlite3_close(p->dbOut);
2031 + assert( res==SQLITE_OK );
2032 + }
2033 + p->dbOut = 0;
2034 +}
2035 +
2036 +/*
2037 +** Decode and return an unsigned 16-bit big-endian integer value from
2038 +** buffer a[].
2039 +*/
2040 +static u32 recoverGetU16(const u8 *a){
2041 + return (((u32)a[0])<<8) + ((u32)a[1]);
2042 +}
2043 +
2044 +/*
2045 +** Decode and return an unsigned 32-bit big-endian integer value from
2046 +** buffer a[].
2047 +*/
2048 +static u32 recoverGetU32(const u8 *a){
2049 + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]);
2050 +}
2051 +
2052 +/*
2053 +** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal)
2054 +** and return the number of bytes consumed.
2055 +*/
2056 +static int recoverGetVarint(const u8 *a, i64 *pVal){
2057 + sqlite3_uint64 u = 0;
2058 + int i;
2059 + for(i=0; i<8; i++){
2060 + u = (u<<7) + (a[i]&0x7f);
2061 + if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
2062 + }
2063 + u = (u<<8) + (a[i]&0xff);
2064 + *pVal = (sqlite3_int64)u;
2065 + return 9;
2066 +}
2067 +
2068 +/*
2069 +** The second argument points to a buffer n bytes in size. If this buffer
2070 +** or a prefix thereof appears to contain a well-formed SQLite b-tree page,
2071 +** return the page-size in bytes. Otherwise, if the buffer does not
2072 +** appear to contain a well-formed b-tree page, return 0.
2073 +*/
2074 +static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){
2075 + u8 *aUsed = aTmp;
2076 + int nFrag = 0;
2077 + int nActual = 0;
2078 + int iFree = 0;
2079 + int nCell = 0; /* Number of cells on page */
2080 + int iCellOff = 0; /* Offset of cell array in page */
2081 + int iContent = 0;
2082 + int eType = 0;
2083 + int ii = 0;
2084 +
2085 + eType = (int)a[0];
2086 + if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0;
2087 +
2088 + iFree = (int)recoverGetU16(&a[1]);
2089 + nCell = (int)recoverGetU16(&a[3]);
2090 + iContent = (int)recoverGetU16(&a[5]);
2091 + if( iContent==0 ) iContent = 65536;
2092 + nFrag = (int)a[7];
2093 +
2094 + if( iContent>n ) return 0;
2095 +
2096 + memset(aUsed, 0, n);
2097 + memset(aUsed, 0xFF, iContent);
2098 +
2099 + /* Follow the free-list. This is the same format for all b-tree pages. */
2100 + if( iFree && iFree<=iContent ) return 0;
2101 + while( iFree ){
2102 + int iNext = 0;
2103 + int nByte = 0;
2104 + if( iFree>(n-4) ) return 0;
2105 + iNext = recoverGetU16(&a[iFree]);
2106 + nByte = recoverGetU16(&a[iFree+2]);
2107 + if( iFree+nByte>n || nByte<4 ) return 0;
2108 + if( iNext && iNext<iFree+nByte ) return 0;
2109 + memset(&aUsed[iFree], 0xFF, nByte);
2110 + iFree = iNext;
2111 + }
2112 +
2113 + /* Run through the cells */
2114 + if( eType==0x02 || eType==0x05 ){
2115 + iCellOff = 12;
2116 + }else{
2117 + iCellOff = 8;
2118 + }
2119 + if( (iCellOff + 2*nCell)>iContent ) return 0;
2120 + for(ii=0; ii<nCell; ii++){
2121 + int iByte;
2122 + i64 nPayload = 0;
2123 + int nByte = 0;
2124 + int iOff = recoverGetU16(&a[iCellOff + 2*ii]);
2125 + if( iOff<iContent || iOff>n ){
2126 + return 0;
2127 + }
2128 + if( eType==0x05 || eType==0x02 ) nByte += 4;
2129 + nByte += recoverGetVarint(&a[iOff+nByte], &nPayload);
2130 + if( eType==0x0D ){
2131 + i64 dummy = 0;
2132 + nByte += recoverGetVarint(&a[iOff+nByte], &dummy);
2133 + }
2134 + if( eType!=0x05 ){
2135 + int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23);
2136 + int M = ((n-12)*32/255)-23;
2137 + int K = M+((nPayload-M)%(n-4));
2138 +
2139 + if( nPayload<X ){
2140 + nByte += nPayload;
2141 + }else if( K<=X ){
2142 + nByte += K+4;
2143 + }else{
2144 + nByte += M+4;
2145 + }
2146 + }
2147 +
2148 + if( iOff+nByte>n ){
2149 + return 0;
2150 + }
2151 + for(iByte=iOff; iByte<(iOff+nByte); iByte++){
2152 + if( aUsed[iByte]!=0 ){
2153 + return 0;
2154 + }
2155 + aUsed[iByte] = 0xFF;
2156 + }
2157 + }
2158 +
2159 + nActual = 0;
2160 + for(ii=0; ii<n; ii++){
2161 + if( aUsed[ii]==0 ) nActual++;
2162 + }
2163 + return (nActual==nFrag);
2164 +}
2165 +
2166 +
2167 +static int recoverVfsClose(sqlite3_file*);
2168 +static int recoverVfsRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
2169 +static int recoverVfsWrite(sqlite3_file*, const void*, int, sqlite3_int64);
2170 +static int recoverVfsTruncate(sqlite3_file*, sqlite3_int64 size);
2171 +static int recoverVfsSync(sqlite3_file*, int flags);
2172 +static int recoverVfsFileSize(sqlite3_file*, sqlite3_int64 *pSize);
2173 +static int recoverVfsLock(sqlite3_file*, int);
2174 +static int recoverVfsUnlock(sqlite3_file*, int);
2175 +static int recoverVfsCheckReservedLock(sqlite3_file*, int *pResOut);
2176 +static int recoverVfsFileControl(sqlite3_file*, int op, void *pArg);
2177 +static int recoverVfsSectorSize(sqlite3_file*);
2178 +static int recoverVfsDeviceCharacteristics(sqlite3_file*);
2179 +static int recoverVfsShmMap(sqlite3_file*, int, int, int, void volatile**);
2180 +static int recoverVfsShmLock(sqlite3_file*, int offset, int n, int flags);
2181 +static void recoverVfsShmBarrier(sqlite3_file*);
2182 +static int recoverVfsShmUnmap(sqlite3_file*, int deleteFlag);
2183 +static int recoverVfsFetch(sqlite3_file*, sqlite3_int64, int, void**);
2184 +static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p);
2185 +
2186 +static sqlite3_io_methods recover_methods = {
2187 + 2, /* iVersion */
2188 + recoverVfsClose,
2189 + recoverVfsRead,
2190 + recoverVfsWrite,
2191 + recoverVfsTruncate,
2192 + recoverVfsSync,
2193 + recoverVfsFileSize,
2194 + recoverVfsLock,
2195 + recoverVfsUnlock,
2196 + recoverVfsCheckReservedLock,
2197 + recoverVfsFileControl,
2198 + recoverVfsSectorSize,
2199 + recoverVfsDeviceCharacteristics,
2200 + recoverVfsShmMap,
2201 + recoverVfsShmLock,
2202 + recoverVfsShmBarrier,
2203 + recoverVfsShmUnmap,
2204 + recoverVfsFetch,
2205 + recoverVfsUnfetch
2206 +};
2207 +
2208 +static int recoverVfsClose(sqlite3_file *pFd){
2209 + assert( pFd->pMethods!=&recover_methods );
2210 + return pFd->pMethods->xClose(pFd);
2211 +}
2212 +
2213 +/*
2214 +** Write value v to buffer a[] as a 16-bit big-endian unsigned integer.
2215 +*/
2216 +static void recoverPutU16(u8 *a, u32 v){
2217 + a[0] = (v>>8) & 0x00FF;
2218 + a[1] = (v>>0) & 0x00FF;
2219 +}
2220 +
2221 +/*
2222 +** Write value v to buffer a[] as a 32-bit big-endian unsigned integer.
2223 +*/
2224 +static void recoverPutU32(u8 *a, u32 v){
2225 + a[0] = (v>>24) & 0x00FF;
2226 + a[1] = (v>>16) & 0x00FF;
2227 + a[2] = (v>>8) & 0x00FF;
2228 + a[3] = (v>>0) & 0x00FF;
2229 +}
2230 +
2231 +/*
2232 +** Detect the page-size of the database opened by file-handle pFd by
2233 +** searching the first part of the file for a well-formed SQLite b-tree
2234 +** page. If parameter nReserve is non-zero, then as well as searching for
2235 +** a b-tree page with zero reserved bytes, this function searches for one
2236 +** with nReserve reserved bytes at the end of it.
2237 +**
2238 +** If successful, set variable p->detected_pgsz to the detected page-size
2239 +** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page
2240 +** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or,
2241 +** if an error occurs (e.g. an IO or OOM error), then an SQLite error code
2242 +** is returned. The final value of p->detected_pgsz is undefined in this
2243 +** case.
2244 +*/
2245 +static int recoverVfsDetectPagesize(
2246 + sqlite3_recover *p, /* Recover handle */
2247 + sqlite3_file *pFd, /* File-handle open on input database */
2248 + u32 nReserve, /* Possible nReserve value */
2249 + i64 nSz /* Size of database file in bytes */
2250 +){
2251 + int rc = SQLITE_OK;
2252 + const int nMin = 512;
2253 + const int nMax = 65536;
2254 + const int nMaxBlk = 4;
2255 + u32 pgsz = 0;
2256 + int iBlk = 0;
2257 + u8 *aPg = 0;
2258 + u8 *aTmp = 0;
2259 + int nBlk = 0;
2260 +
2261 + aPg = (u8*)sqlite3_malloc(2*nMax);
2262 + if( aPg==0 ) return SQLITE_NOMEM;
2263 + aTmp = &aPg[nMax];
2264 +
2265 + nBlk = (nSz+nMax-1)/nMax;
2266 + if( nBlk>nMaxBlk ) nBlk = nMaxBlk;
2267 +
2268 + do {
2269 + for(iBlk=0; rc==SQLITE_OK && iBlk<nBlk; iBlk++){
2270 + int nByte = (nSz>=((iBlk+1)*nMax)) ? nMax : (nSz % nMax);
2271 + memset(aPg, 0, nMax);
2272 + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax);
2273 + if( rc==SQLITE_OK ){
2274 + int pgsz2;
2275 + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){
2276 + int iOff;
2277 + for(iOff=0; iOff<nMax; iOff+=pgsz2){
2278 + if( recoverIsValidPage(aTmp, &aPg[iOff], pgsz2-nReserve) ){
2279 + pgsz = pgsz2;
2280 + break;
2281 + }
2282 + }
2283 + }
2284 + }
2285 + }
2286 + if( pgsz>(u32)p->detected_pgsz ){
2287 + p->detected_pgsz = pgsz;
2288 + p->nReserve = nReserve;
2289 + }
2290 + if( nReserve==0 ) break;
2291 + nReserve = 0;
2292 + }while( 1 );
2293 +
2294 + p->detected_pgsz = pgsz;
2295 + sqlite3_free(aPg);
2296 + return rc;
2297 +}
2298 +
2299 +/*
2300 +** The xRead() method of the wrapper VFS. This is used to intercept calls
2301 +** to read page 1 of the input database.
2302 +*/
2303 +static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){
2304 + int rc = SQLITE_OK;
2305 + if( pFd->pMethods==&recover_methods ){
2306 + pFd->pMethods = recover_g.pMethods;
2307 + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff);
2308 + if( nByte==16 ){
2309 + sqlite3_randomness(16, aBuf);
2310 + }else
2311 + if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){
2312 + /* Ensure that the database has a valid header file. The only fields
2313 + ** that really matter to recovery are:
2314 + **
2315 + ** + Database page size (16-bits at offset 16)
2316 + ** + Size of db in pages (32-bits at offset 28)
2317 + ** + Database encoding (32-bits at offset 56)
2318 + **
2319 + ** Also preserved are:
2320 + **
2321 + ** + first freelist page (32-bits at offset 32)
2322 + ** + size of freelist (32-bits at offset 36)
2323 + ** + the wal-mode flags (16-bits at offset 18)
2324 + **
2325 + ** We also try to preserve the auto-vacuum, incr-value, user-version
2326 + ** and application-id fields - all 32 bit quantities at offsets
2327 + ** 52, 60, 64 and 68. All other fields are set to known good values.
2328 + **
2329 + ** Byte offset 105 should also contain the page-size as a 16-bit
2330 + ** integer.
2331 + */
2332 + const int aPreserve[] = {32, 36, 52, 60, 64, 68};
2333 + u8 aHdr[108] = {
2334 + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66,
2335 + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
2336 + 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20,
2337 + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF,
2338 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
2339 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
2340 + 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF,
2341 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
2342 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
2343 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2344 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2345 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2346 + 0x00, 0x2e, 0x5b, 0x30,
2347 +
2348 + 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00
2349 + };
2350 + u8 *a = (u8*)aBuf;
2351 +
2352 + u32 pgsz = recoverGetU16(&a[16]);
2353 + u32 nReserve = a[20];
2354 + u32 enc = recoverGetU32(&a[56]);
2355 + u32 dbsz = 0;
2356 + i64 dbFileSize = 0;
2357 + int ii;
2358 + sqlite3_recover *p = recover_g.p;
2359 +
2360 + if( pgsz==0x01 ) pgsz = 65536;
2361 + rc = pFd->pMethods->xFileSize(pFd, &dbFileSize);
2362 +
2363 + if( rc==SQLITE_OK && p->detected_pgsz==0 ){
2364 + rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize);
2365 + }
2366 + if( p->detected_pgsz ){
2367 + pgsz = p->detected_pgsz;
2368 + nReserve = p->nReserve;
2369 + }
2370 +
2371 + if( pgsz ){
2372 + dbsz = dbFileSize / pgsz;
2373 + }
2374 + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){
2375 + enc = SQLITE_UTF8;
2376 + }
2377 +
2378 + sqlite3_free(p->pPage1Cache);
2379 + p->pPage1Cache = 0;
2380 + p->pPage1Disk = 0;
2381 +
2382 + p->pgsz = nByte;
2383 + p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2);
2384 + if( p->pPage1Cache ){
2385 + p->pPage1Disk = &p->pPage1Cache[nByte];
2386 + memcpy(p->pPage1Disk, aBuf, nByte);
2387 + aHdr[18] = a[18];
2388 + aHdr[19] = a[19];
2389 + recoverPutU32(&aHdr[28], dbsz);
2390 + recoverPutU32(&aHdr[56], enc);
2391 + recoverPutU16(&aHdr[105], pgsz-nReserve);
2392 + if( pgsz==65536 ) pgsz = 1;
2393 + recoverPutU16(&aHdr[16], pgsz);
2394 + aHdr[20] = nReserve;
2395 + for(ii=0; ii<(int)(sizeof(aPreserve)/sizeof(aPreserve[0])); ii++){
2396 + memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4);
2397 + }
2398 + memcpy(aBuf, aHdr, sizeof(aHdr));
2399 + memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr));
2400 +
2401 + memcpy(p->pPage1Cache, aBuf, nByte);
2402 + }else{
2403 + rc = p->errCode;
2404 + }
2405 +
2406 + }
2407 + pFd->pMethods = &recover_methods;
2408 + }else{
2409 + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff);
2410 + }
2411 + return rc;
2412 +}
2413 +
2414 +/*
2415 +** Used to make sqlite3_io_methods wrapper methods less verbose.
2416 +*/
2417 +#define RECOVER_VFS_WRAPPER(code) \
2418 + int rc = SQLITE_OK; \
2419 + if( pFd->pMethods==&recover_methods ){ \
2420 + pFd->pMethods = recover_g.pMethods; \
2421 + rc = code; \
2422 + pFd->pMethods = &recover_methods; \
2423 + }else{ \
2424 + rc = code; \
2425 + } \
2426 + return rc;
2427 +
2428 +/*
2429 +** Methods of the wrapper VFS. All methods except for xRead() and xClose()
2430 +** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent
2431 +** method on the lower level VFS, then reinstall the wrapper before returning.
2432 +** Those that return an integer value use the RECOVER_VFS_WRAPPER macro.
2433 +*/
2434 +static int recoverVfsWrite(
2435 + sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff
2436 +){
2437 + RECOVER_VFS_WRAPPER (
2438 + pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff)
2439 + );
2440 +}
2441 +static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){
2442 + RECOVER_VFS_WRAPPER (
2443 + pFd->pMethods->xTruncate(pFd, size)
2444 + );
2445 +}
2446 +static int recoverVfsSync(sqlite3_file *pFd, int flags){
2447 + RECOVER_VFS_WRAPPER (
2448 + pFd->pMethods->xSync(pFd, flags)
2449 + );
2450 +}
2451 +static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){
2452 + RECOVER_VFS_WRAPPER (
2453 + pFd->pMethods->xFileSize(pFd, pSize)
2454 + );
2455 +}
2456 +static int recoverVfsLock(sqlite3_file *pFd, int eLock){
2457 + RECOVER_VFS_WRAPPER (
2458 + pFd->pMethods->xLock(pFd, eLock)
2459 + );
2460 +}
2461 +static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){
2462 + RECOVER_VFS_WRAPPER (
2463 + pFd->pMethods->xUnlock(pFd, eLock)
2464 + );
2465 +}
2466 +static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){
2467 + RECOVER_VFS_WRAPPER (
2468 + pFd->pMethods->xCheckReservedLock(pFd, pResOut)
2469 + );
2470 +}
2471 +static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){
2472 + RECOVER_VFS_WRAPPER (
2473 + (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND)
2474 + );
2475 +}
2476 +static int recoverVfsSectorSize(sqlite3_file *pFd){
2477 + RECOVER_VFS_WRAPPER (
2478 + pFd->pMethods->xSectorSize(pFd)
2479 + );
2480 +}
2481 +static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){
2482 + RECOVER_VFS_WRAPPER (
2483 + pFd->pMethods->xDeviceCharacteristics(pFd)
2484 + );
2485 +}
2486 +static int recoverVfsShmMap(
2487 + sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp
2488 +){
2489 + RECOVER_VFS_WRAPPER (
2490 + pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp)
2491 + );
2492 +}
2493 +static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){
2494 + RECOVER_VFS_WRAPPER (
2495 + pFd->pMethods->xShmLock(pFd, offset, n, flags)
2496 + );
2497 +}
2498 +static void recoverVfsShmBarrier(sqlite3_file *pFd){
2499 + if( pFd->pMethods==&recover_methods ){
2500 + pFd->pMethods = recover_g.pMethods;
2501 + pFd->pMethods->xShmBarrier(pFd);
2502 + pFd->pMethods = &recover_methods;
2503 + }else{
2504 + pFd->pMethods->xShmBarrier(pFd);
2505 + }
2506 +}
2507 +static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){
2508 + RECOVER_VFS_WRAPPER (
2509 + pFd->pMethods->xShmUnmap(pFd, deleteFlag)
2510 + );
2511 +}
2512 +
2513 +static int recoverVfsFetch(
2514 + sqlite3_file *pFd,
2515 + sqlite3_int64 iOff,
2516 + int iAmt,
2517 + void **pp
2518 +){
2519 + (void)pFd;
2520 + (void)iOff;
2521 + (void)iAmt;
2522 + *pp = 0;
2523 + return SQLITE_OK;
2524 +}
2525 +static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){
2526 + (void)pFd;
2527 + (void)iOff;
2528 + (void)p;
2529 + return SQLITE_OK;
2530 +}
2531 +
2532 +/*
2533 +** Install the VFS wrapper around the file-descriptor open on the input
2534 +** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held
2535 +** when this function is called.
2536 +*/
2537 +static void recoverInstallWrapper(sqlite3_recover *p){
2538 + sqlite3_file *pFd = 0;
2539 + assert( recover_g.pMethods==0 );
2540 + recoverAssertMutexHeld();
2541 + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd);
2542 + assert( pFd==0 || pFd->pMethods!=&recover_methods );
2543 + if( pFd && pFd->pMethods ){
2544 + int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0);
2545 + recover_g.pMethods = pFd->pMethods;
2546 + recover_g.p = p;
2547 + recover_methods.iVersion = iVersion;
2548 + pFd->pMethods = &recover_methods;
2549 + }
2550 +}
2551 +
2552 +/*
2553 +** Uninstall the VFS wrapper that was installed around the file-descriptor open
2554 +** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be
2555 +** held when this function is called.
2556 +*/
2557 +static void recoverUninstallWrapper(sqlite3_recover *p){
2558 + sqlite3_file *pFd = 0;
2559 + recoverAssertMutexHeld();
2560 + sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd);
2561 + if( pFd && pFd->pMethods ){
2562 + pFd->pMethods = recover_g.pMethods;
2563 + recover_g.pMethods = 0;
2564 + recover_g.p = 0;
2565 + }
2566 +}
2567 +
2568 +/*
2569 +** This function does the work of a single sqlite3_recover_step() call. It
2570 +** is guaranteed that the handle is not in an error state when this
2571 +** function is called.
2572 +*/
2573 +static void recoverStep(sqlite3_recover *p){
2574 + assert( p && p->errCode==SQLITE_OK );
2575 + switch( p->eState ){
2576 + case RECOVER_STATE_INIT:
2577 + /* This is the very first call to sqlite3_recover_step() on this object.
2578 + */
2579 + recoverSqlCallback(p, "BEGIN");
2580 + recoverSqlCallback(p, "PRAGMA writable_schema = on");
2581 +
2582 + recoverEnterMutex();
2583 + recoverInstallWrapper(p);
2584 +
2585 + /* Open the output database. And register required virtual tables and
2586 + ** user functions with the new handle. */
2587 + recoverOpenOutput(p);
2588 +
2589 + /* Open transactions on both the input and output databases. */
2590 + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0);
2591 + recoverExec(p, p->dbIn, "PRAGMA writable_schema = on");
2592 + recoverExec(p, p->dbIn, "BEGIN");
2593 + if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1;
2594 + recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema");
2595 + recoverTransferSettings(p);
2596 + recoverOpenRecovery(p);
2597 + recoverCacheSchema(p);
2598 +
2599 + recoverUninstallWrapper(p);
2600 + recoverLeaveMutex();
2601 +
2602 + recoverExec(p, p->dbOut, "BEGIN");
2603 +
2604 + recoverWriteSchema1(p);
2605 + p->eState = RECOVER_STATE_WRITING;
2606 + break;
2607 +
2608 + case RECOVER_STATE_WRITING: {
2609 + if( p->w1.pTbls==0 ){
2610 + recoverWriteDataInit(p);
2611 + }
2612 + if( SQLITE_DONE==recoverWriteDataStep(p) ){
2613 + recoverWriteDataCleanup(p);
2614 + if( p->zLostAndFound ){
2615 + p->eState = RECOVER_STATE_LOSTANDFOUND1;
2616 + }else{
2617 + p->eState = RECOVER_STATE_SCHEMA2;
2618 + }
2619 + }
2620 + break;
2621 + }
2622 +
2623 + case RECOVER_STATE_LOSTANDFOUND1: {
2624 + if( p->laf.pUsed==0 ){
2625 + recoverLostAndFound1Init(p);
2626 + }
2627 + if( SQLITE_DONE==recoverLostAndFound1Step(p) ){
2628 + p->eState = RECOVER_STATE_LOSTANDFOUND2;
2629 + }
2630 + break;
2631 + }
2632 + case RECOVER_STATE_LOSTANDFOUND2: {
2633 + if( p->laf.pAllAndParent==0 ){
2634 + recoverLostAndFound2Init(p);
2635 + }
2636 + if( SQLITE_DONE==recoverLostAndFound2Step(p) ){
2637 + p->eState = RECOVER_STATE_LOSTANDFOUND3;
2638 + }
2639 + break;
2640 + }
2641 +
2642 + case RECOVER_STATE_LOSTANDFOUND3: {
2643 + if( p->laf.pInsert==0 ){
2644 + recoverLostAndFound3Init(p);
2645 + }
2646 + if( SQLITE_DONE==recoverLostAndFound3Step(p) ){
2647 + p->eState = RECOVER_STATE_SCHEMA2;
2648 + }
2649 + break;
2650 + }
2651 +
2652 + case RECOVER_STATE_SCHEMA2: {
2653 + int rc = SQLITE_OK;
2654 +
2655 + recoverWriteSchema2(p);
2656 + p->eState = RECOVER_STATE_DONE;
2657 +
2658 + /* If no error has occurred, commit the write transaction on the output
2659 + ** database. Regardless of whether or not an error has occurred, make
2660 + ** an attempt to end the read transaction on the input database. */
2661 + recoverExec(p, p->dbOut, "COMMIT");
2662 + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0);
2663 + if( p->errCode==SQLITE_OK ) p->errCode = rc;
2664 +
2665 + recoverSqlCallback(p, "PRAGMA writable_schema = off");
2666 + recoverSqlCallback(p, "COMMIT");
2667 + p->eState = RECOVER_STATE_DONE;
2668 + recoverFinalCleanup(p);
2669 + break;
2670 + };
2671 +
2672 + case RECOVER_STATE_DONE: {
2673 + /* no-op */
2674 + break;
2675 + };
2676 + }
2677 +}
2678 +
2679 +
2680 +/*
2681 +** This is a worker function that does the heavy lifting for both init
2682 +** functions:
2683 +**
2684 +** sqlite3_recover_init()
2685 +** sqlite3_recover_init_sql()
2686 +**
2687 +** All this function does is allocate space for the recover handle and
2688 +** take copies of the input parameters. All the real work is done within
2689 +** sqlite3_recover_run().
2690 +*/
2691 +sqlite3_recover *recoverInit(
2692 + sqlite3* db,
2693 + const char *zDb,
2694 + const char *zUri, /* Output URI for _recover_init() */
2695 + int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */
2696 + void *pSqlCtx /* Context arg for _recover_init_sql() */
2697 +){
2698 + sqlite3_recover *pRet = 0;
2699 + int nDb = 0;
2700 + int nUri = 0;
2701 + int nByte = 0;
2702 +
2703 + if( zDb==0 ){ zDb = "main"; }
2704 +
2705 + nDb = recoverStrlen(zDb);
2706 + nUri = recoverStrlen(zUri);
2707 +
2708 + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1;
2709 + pRet = (sqlite3_recover*)sqlite3_malloc(nByte);
2710 + if( pRet ){
2711 + memset(pRet, 0, nByte);
2712 + pRet->dbIn = db;
2713 + pRet->zDb = (char*)&pRet[1];
2714 + pRet->zUri = &pRet->zDb[nDb+1];
2715 + memcpy(pRet->zDb, zDb, nDb);
2716 + if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri);
2717 + pRet->xSql = xSql;
2718 + pRet->pSqlCtx = pSqlCtx;
2719 + pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT;
2720 + }
2721 +
2722 + return pRet;
2723 +}
2724 +
2725 +/*
2726 +** Initialize a recovery handle that creates a new database containing
2727 +** the recovered data.
2728 +*/
2729 +sqlite3_recover *sqlite3_recover_init(
2730 + sqlite3* db,
2731 + const char *zDb,
2732 + const char *zUri
2733 +){
2734 + return recoverInit(db, zDb, zUri, 0, 0);
2735 +}
2736 +
2737 +/*
2738 +** Initialize a recovery handle that returns recovered data in the
2739 +** form of SQL statements via a callback.
2740 +*/
2741 +sqlite3_recover *sqlite3_recover_init_sql(
2742 + sqlite3* db,
2743 + const char *zDb,
2744 + int (*xSql)(void*, const char*),
2745 + void *pSqlCtx
2746 +){
2747 + return recoverInit(db, zDb, 0, xSql, pSqlCtx);
2748 +}
2749 +
2750 +/*
2751 +** Return the handle error message, if any.
2752 +*/
2753 +const char *sqlite3_recover_errmsg(sqlite3_recover *p){
2754 + return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory";
2755 +}
2756 +
2757 +/*
2758 +** Return the handle error code.
2759 +*/
2760 +int sqlite3_recover_errcode(sqlite3_recover *p){
2761 + return p ? p->errCode : SQLITE_NOMEM;
2762 +}
2763 +
2764 +/*
2765 +** Configure the handle.
2766 +*/
2767 +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){
2768 + int rc = SQLITE_OK;
2769 + if( p==0 ){
2770 + rc = SQLITE_NOMEM;
2771 + }else if( p->eState!=RECOVER_STATE_INIT ){
2772 + rc = SQLITE_MISUSE;
2773 + }else{
2774 + switch( op ){
2775 + case 789:
2776 + /* This undocumented magic configuration option is used to set the
2777 + ** name of the auxiliary database that is ATTACH-ed to the database
2778 + ** connection and used to hold state information during the
2779 + ** recovery process. This option is for debugging use only and
2780 + ** is subject to change or removal at any time. */
2781 + sqlite3_free(p->zStateDb);
2782 + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg);
2783 + break;
2784 +
2785 + case SQLITE_RECOVER_LOST_AND_FOUND: {
2786 + const char *zArg = (const char*)pArg;
2787 + sqlite3_free(p->zLostAndFound);
2788 + if( zArg ){
2789 + p->zLostAndFound = recoverMPrintf(p, "%s", zArg);
2790 + }else{
2791 + p->zLostAndFound = 0;
2792 + }
2793 + break;
2794 + }
2795 +
2796 + case SQLITE_RECOVER_FREELIST_CORRUPT:
2797 + p->bFreelistCorrupt = *(int*)pArg;
2798 + break;
2799 +
2800 + case SQLITE_RECOVER_ROWIDS:
2801 + p->bRecoverRowid = *(int*)pArg;
2802 + break;
2803 +
2804 + case SQLITE_RECOVER_SLOWINDEXES:
2805 + p->bSlowIndexes = *(int*)pArg;
2806 + break;
2807 +
2808 + default:
2809 + rc = SQLITE_NOTFOUND;
2810 + break;
2811 + }
2812 + }
2813 +
2814 + return rc;
2815 +}
2816 +
2817 +/*
2818 +** Do a unit of work towards the recovery job. Return SQLITE_OK if
2819 +** no error has occurred but database recovery is not finished, SQLITE_DONE
2820 +** if database recovery has been successfully completed, or an SQLite
2821 +** error code if an error has occurred.
2822 +*/
2823 +int sqlite3_recover_step(sqlite3_recover *p){
2824 + if( p==0 ) return SQLITE_NOMEM;
2825 + if( p->errCode==SQLITE_OK ) recoverStep(p);
2826 + if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){
2827 + return SQLITE_DONE;
2828 + }
2829 + return p->errCode;
2830 +}
2831 +
2832 +/*
2833 +** Do the configured recovery operation. Return SQLITE_OK if successful, or
2834 +** else an SQLite error code.
2835 +*/
2836 +int sqlite3_recover_run(sqlite3_recover *p){
2837 + while( SQLITE_OK==sqlite3_recover_step(p) );
2838 + return sqlite3_recover_errcode(p);
2839 +}
2840 +
2841 +
2842 +/*
2843 +** Free all resources associated with the recover handle passed as the only
2844 +** argument. The results of using a handle with any sqlite3_recover_**
2845 +** API function after it has been passed to this function are undefined.
2846 +**
2847 +** A copy of the value returned by the first call made to sqlite3_recover_run()
2848 +** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has
2849 +** not been called on this handle.
2850 +*/
2851 +int sqlite3_recover_finish(sqlite3_recover *p){
2852 + int rc;
2853 + if( p==0 ){
2854 + rc = SQLITE_NOMEM;
2855 + }else{
2856 + recoverFinalCleanup(p);
2857 + if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){
2858 + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0);
2859 + if( p->errCode==SQLITE_OK ) p->errCode = rc;
2860 + }
2861 + rc = p->errCode;
2862 + sqlite3_free(p->zErrMsg);
2863 + sqlite3_free(p->zStateDb);
2864 + sqlite3_free(p->zLostAndFound);
2865 + sqlite3_free(p->pPage1Cache);
2866 + sqlite3_free(p);
2867 + }
2868 + return rc;
2869 +}
2870 +
2871 +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
2872 +#pragma GCC diagnostic pop
database/sqlite/sqlite3recover.h new
+249
@@ -0,0 +1,249 @@
1 +/*
2 +** 2022-08-27
3 +**
4 +** The author disclaims copyright to this source code. In place of
5 +** a legal notice, here is a blessing:
6 +**
7 +** May you do good and not evil.
8 +** May you find forgiveness for yourself and forgive others.
9 +** May you share freely, never taking more than you give.
10 +**
11 +*************************************************************************
12 +**
13 +** This file contains the public interface to the "recover" extension -
14 +** an SQLite extension designed to recover data from corrupted database
15 +** files.
16 +*/
17 +
18 +/*
19 +** OVERVIEW:
20 +**
21 +** To use the API to recover data from a corrupted database, an
22 +** application:
23 +**
24 +** 1) Creates an sqlite3_recover handle by calling either
25 +** sqlite3_recover_init() or sqlite3_recover_init_sql().
26 +**
27 +** 2) Configures the new handle using one or more calls to
28 +** sqlite3_recover_config().
29 +**
30 +** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on
31 +** the handle until it returns something other than SQLITE_OK. If it
32 +** returns SQLITE_DONE, then the recovery operation completed without
33 +** error. If it returns some other non-SQLITE_OK value, then an error
34 +** has occurred.
35 +**
36 +** 4) Retrieves any error code and English language error message using the
37 +** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs,
38 +** respectively.
39 +**
40 +** 5) Destroys the sqlite3_recover handle and frees all resources
41 +** using sqlite3_recover_finish().
42 +**
43 +** The application may abandon the recovery operation at any point
44 +** before it is finished by passing the sqlite3_recover handle to
45 +** sqlite3_recover_finish(). This is not an error, but the final state
46 +** of the output database, or the results of running the partial script
47 +** delivered to the SQL callback, are undefined.
48 +*/
49 +
50 +#ifndef _SQLITE_RECOVER_H
51 +#define _SQLITE_RECOVER_H
52 +
53 +#include "sqlite3.h"
54 +
55 +#ifdef __cplusplus
56 +extern "C" {
57 +#endif
58 +
59 +/*
60 +** An instance of the sqlite3_recover object represents a recovery
61 +** operation in progress.
62 +**
63 +** Constructors:
64 +**
65 +** sqlite3_recover_init()
66 +** sqlite3_recover_init_sql()
67 +**
68 +** Destructor:
69 +**
70 +** sqlite3_recover_finish()
71 +**
72 +** Methods:
73 +**
74 +** sqlite3_recover_config()
75 +** sqlite3_recover_errcode()
76 +** sqlite3_recover_errmsg()
77 +** sqlite3_recover_run()
78 +** sqlite3_recover_step()
79 +*/
80 +typedef struct sqlite3_recover sqlite3_recover;
81 +
82 +/*
83 +** These two APIs attempt to create and return a new sqlite3_recover object.
84 +** In both cases the first two arguments identify the (possibly
85 +** corrupt) database to recover data from. The first argument is an open
86 +** database handle and the second the name of a database attached to that
87 +** handle (i.e. "main", "temp" or the name of an attached database).
88 +**
89 +** If sqlite3_recover_init() is used to create the new sqlite3_recover
90 +** handle, then data is recovered into a new database, identified by
91 +** string parameter zUri. zUri may be an absolute or relative file path,
92 +** or may be an SQLite URI. If the identified database file already exists,
93 +** it is overwritten.
94 +**
95 +** If sqlite3_recover_init_sql() is invoked, then any recovered data will
96 +** be returned to the user as a series of SQL statements. Executing these
97 +** SQL statements results in the same database as would have been created
98 +** had sqlite3_recover_init() been used. For each SQL statement in the
99 +** output, the callback function passed as the third argument (xSql) is
100 +** invoked once. The first parameter is a passed a copy of the fourth argument
101 +** to this function (pCtx) as its first parameter, and a pointer to a
102 +** nul-terminated buffer containing the SQL statement formated as UTF-8 as
103 +** the second. If the xSql callback returns any value other than SQLITE_OK,
104 +** then processing is immediately abandoned and the value returned used as
105 +** the recover handle error code (see below).
106 +**
107 +** If an out-of-memory error occurs, NULL may be returned instead of
108 +** a valid handle. In all other cases, it is the responsibility of the
109 +** application to avoid resource leaks by ensuring that
110 +** sqlite3_recover_finish() is called on all allocated handles.
111 +*/
112 +sqlite3_recover *sqlite3_recover_init(
113 + sqlite3* db,
114 + const char *zDb,
115 + const char *zUri
116 +);
117 +sqlite3_recover *sqlite3_recover_init_sql(
118 + sqlite3* db,
119 + const char *zDb,
120 + int (*xSql)(void*, const char*),
121 + void *pCtx
122 +);
123 +
124 +/*
125 +** Configure an sqlite3_recover object that has just been created using
126 +** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function
127 +** may only be called before the first call to sqlite3_recover_step()
128 +** or sqlite3_recover_run() on the object.
129 +**
130 +** The second argument passed to this function must be one of the
131 +** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument
132 +** depend on the specific SQLITE_RECOVER_* symbol in use.
133 +**
134 +** SQLITE_OK is returned if the configuration operation was successful,
135 +** or an SQLite error code otherwise.
136 +*/
137 +int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg);
138 +
139 +/*
140 +** SQLITE_RECOVER_LOST_AND_FOUND:
141 +** The pArg argument points to a string buffer containing the name
142 +** of a "lost-and-found" table in the output database, or NULL. If
143 +** the argument is non-NULL and the database contains seemingly
144 +** valid pages that cannot be associated with any table in the
145 +** recovered part of the schema, data is extracted from these
146 +** pages to add to the lost-and-found table.
147 +**
148 +** SQLITE_RECOVER_FREELIST_CORRUPT:
149 +** The pArg value must actually be a pointer to a value of type
150 +** int containing value 0 or 1 cast as a (void*). If this option is set
151 +** (argument is 1) and a lost-and-found table has been configured using
152 +** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is
153 +** corrupt and an attempt is made to recover records from pages that
154 +** appear to be linked into the freelist. Otherwise, pages on the freelist
155 +** are ignored. Setting this option can recover more data from the
156 +** database, but often ends up "recovering" deleted records. The default
157 +** value is 0 (clear).
158 +**
159 +** SQLITE_RECOVER_ROWIDS:
160 +** The pArg value must actually be a pointer to a value of type
161 +** int containing value 0 or 1 cast as a (void*). If this option is set
162 +** (argument is 1), then an attempt is made to recover rowid values
163 +** that are not also INTEGER PRIMARY KEY values. If this option is
164 +** clear, then new rowids are assigned to all recovered rows. The
165 +** default value is 1 (set).
166 +**
167 +** SQLITE_RECOVER_SLOWINDEXES:
168 +** The pArg value must actually be a pointer to a value of type
169 +** int containing value 0 or 1 cast as a (void*). If this option is clear
170 +** (argument is 0), then when creating an output database, the recover
171 +** module creates and populates non-UNIQUE indexes right at the end of the
172 +** recovery operation - after all recoverable data has been inserted
173 +** into the new database. This is faster overall, but means that the
174 +** final call to sqlite3_recover_step() for a recovery operation may
175 +** be need to create a large number of indexes, which may be very slow.
176 +**
177 +** Or, if this option is set (argument is 1), then non-UNIQUE indexes
178 +** are created in the output database before it is populated with
179 +** recovered data. This is slower overall, but avoids the slow call
180 +** to sqlite3_recover_step() at the end of the recovery operation.
181 +**
182 +** The default option value is 0.
183 +*/
184 +#define SQLITE_RECOVER_LOST_AND_FOUND 1
185 +#define SQLITE_RECOVER_FREELIST_CORRUPT 2
186 +#define SQLITE_RECOVER_ROWIDS 3
187 +#define SQLITE_RECOVER_SLOWINDEXES 4
188 +
189 +/*
190 +** Perform a unit of work towards the recovery operation. This function
191 +** must normally be called multiple times to complete database recovery.
192 +**
193 +** If no error occurs but the recovery operation is not completed, this
194 +** function returns SQLITE_OK. If recovery has been completed successfully
195 +** then SQLITE_DONE is returned. If an error has occurred, then an SQLite
196 +** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not
197 +** considered an error if some or all of the data cannot be recovered
198 +** due to database corruption.
199 +**
200 +** Once sqlite3_recover_step() has returned a value other than SQLITE_OK,
201 +** all further such calls on the same recover handle are no-ops that return
202 +** the same non-SQLITE_OK value.
203 +*/
204 +int sqlite3_recover_step(sqlite3_recover*);
205 +
206 +/*
207 +** Run the recovery operation to completion. Return SQLITE_OK if successful,
208 +** or an SQLite error code otherwise. Calling this function is the same
209 +** as executing:
210 +**
211 +** while( SQLITE_OK==sqlite3_recover_step(p) );
212 +** return sqlite3_recover_errcode(p);
213 +*/
214 +int sqlite3_recover_run(sqlite3_recover*);
215 +
216 +/*
217 +** If an error has been encountered during a prior call to
218 +** sqlite3_recover_step(), then this function attempts to return a
219 +** pointer to a buffer containing an English language explanation of
220 +** the error. If no error message is available, or if an out-of memory
221 +** error occurs while attempting to allocate a buffer in which to format
222 +** the error message, NULL is returned.
223 +**
224 +** The returned buffer remains valid until the sqlite3_recover handle is
225 +** destroyed using sqlite3_recover_finish().
226 +*/
227 +const char *sqlite3_recover_errmsg(sqlite3_recover*);
228 +
229 +/*
230 +** If this function is called on an sqlite3_recover handle after
231 +** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK.
232 +*/
233 +int sqlite3_recover_errcode(sqlite3_recover*);
234 +
235 +/*
236 +** Clean up a recovery object created by a call to sqlite3_recover_init().
237 +** The results of using a recovery object with any API after it has been
238 +** passed to this function are undefined.
239 +**
240 +** This function returns the same value as sqlite3_recover_errcode().
241 +*/
242 +int sqlite3_recover_finish(sqlite3_recover*);
243 +
244 +
245 +#ifdef __cplusplus
246 +} /* end of the 'extern "C"' block */
247 +#endif
248 +
249 +#endif /* ifndef _SQLITE_RECOVER_H */
database/sqlite/sqlite_context.c
+10 -10
@@ -55,48 +55,48 @@ int sql_init_context_database(int memory)
55 // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
56 // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
57 snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
58 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
58 + if(init_database_batch(db_context_meta, list)) return 1;
59
60 // https://www.sqlite.org/pragma.html#pragma_synchronous
61 // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
62 snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
63 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
63 + if(init_database_batch(db_context_meta, list)) return 1;
64
65 // https://www.sqlite.org/pragma.html#pragma_journal_mode
66 // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
67 snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
68 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
68 + if(init_database_batch(db_context_meta, list)) return 1;
69
70 // https://www.sqlite.org/pragma.html#pragma_temp_store
71 // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
72 snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
73 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
73 + if(init_database_batch(db_context_meta, list)) return 1;
74
75 // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
76 // PRAGMA schema.journal_size_limit = N ;
77 snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
78 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
78 + if(init_database_batch(db_context_meta, list)) return 1;
79
80 // https://www.sqlite.org/pragma.html#pragma_cache_size
81 // PRAGMA schema.cache_size = pages;
82 // PRAGMA schema.cache_size = -kibibytes;
83 snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
84 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
84 + if(init_database_batch(db_context_meta, list)) return 1;
85
86 snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
87 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
87 + if(init_database_batch(db_context_meta, list)) return 1;
88
89 if (likely(!memory))
90 snprintfz(buf, 1024, "ATTACH DATABASE \"%s/netdata-meta.db\" as meta;", netdata_configured_cache_dir);
91 else
92 snprintfz(buf, 1024, "ATTACH DATABASE ':memory:' as meta;");
93
94 - if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
94 + if(init_database_batch(db_context_meta, list)) return 1;
95
96 - if (init_database_batch(db_context_meta, DB_CHECK_NONE, 0, &database_context_config[0]))
96 + if (init_database_batch(db_context_meta, &database_context_config[0]))
97 return 1;
98
99 - if (init_database_batch(db_context_meta, DB_CHECK_NONE, 0, &database_context_cleanup[0]))
99 + if (init_database_batch(db_context_meta, &database_context_cleanup[0]))
100 return 1;
101
102 return 0;
database/sqlite/sqlite_db_migration.c
+6 -6
@@ -94,7 +94,7 @@ static int do_migration_v1_v2(sqlite3 *database, const char *name)
94 netdata_log_info("Running \"%s\" database migration", name);
95
96 if (table_exists_in_database("host") && !column_exists_in_table("host", "hops"))
97 - return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v1_v2[0]);
97 + return init_database_batch(database, &database_migrate_v1_v2[0]);
98 return 0;
99 }
100
@@ -104,7 +104,7 @@ static int do_migration_v2_v3(sqlite3 *database, const char *name)
104 netdata_log_info("Running \"%s\" database migration", name);
105
106 if (table_exists_in_database("host") && !column_exists_in_table("host", "memory_mode"))
107 - return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v2_v3[0]);
107 + return init_database_batch(database, &database_migrate_v2_v3[0]);
108 return 0;
109 }
110
@@ -145,7 +145,7 @@ static int do_migration_v4_v5(sqlite3 *database, const char *name)
145 UNUSED(name);
146 netdata_log_info("Running \"%s\" database migration", name);
147
148 - return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v4_v5[0]);
148 + return init_database_batch(database, &database_migrate_v4_v5[0]);
149 }
150
151 static int do_migration_v5_v6(sqlite3 *database, const char *name)
@@ -153,7 +153,7 @@ static int do_migration_v5_v6(sqlite3 *database, const char *name)
153 UNUSED(name);
154 netdata_log_info("Running \"%s\" database migration", name);
155
156 - return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v5_v6[0]);
156 + return init_database_batch(database, &database_migrate_v5_v6[0]);
157 }
158
159 static int do_migration_v6_v7(sqlite3 *database, const char *name)
@@ -301,7 +301,7 @@ static int do_migration_v9_v10(sqlite3 *database, const char *name)
301 netdata_log_info("Running \"%s\" database migration", name);
302
303 if (table_exists_in_database("alert_hash") && !column_exists_in_table("alert_hash", "chart_labels"))
304 - return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v9_v10[0]);
304 + return init_database_batch(database, &database_migrate_v9_v10[0]);
305 return 0;
306 }
307
@@ -310,7 +310,7 @@ static int do_migration_v10_v11(sqlite3 *database, const char *name)
310 netdata_log_info("Running \"%s\" database migration", name);
311
312 if (table_exists_in_database("health_log") && !column_exists_in_table("health_log", "chart_name"))
313 - return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v10_v11[0]);
313 + return init_database_batch(database, &database_migrate_v10_v11[0]);
314
315 return 0;
316 }
database/sqlite/sqlite_functions.c
+95 -160
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "sqlite_functions.h"
4 +#include "sqlite3recover.h"
5 #include "sqlite_db_migration.h"
6
7 #define DB_METADATA_VERSION 11
@@ -120,6 +121,66 @@ SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt) {
121 return rc;
122 }
123
124 +static bool mark_database_to_recover(sqlite3_stmt *res, sqlite3 *database)
125 +{
126 +
127 + if (!res && !database)
128 + return false;
129 +
130 + if (!database)
131 + database = sqlite3_db_handle(res);
132 +
133 + if (db_meta == database) {
134 + char recover_file[FILENAME_MAX + 1];
135 + snprintfz(recover_file, FILENAME_MAX, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);
136 + int fd = open(recover_file, O_WRONLY | O_CREAT | O_TRUNC, 444);
137 + if (fd >= 0) {
138 + close(fd);
139 + return true;
140 + }
141 + }
142 + return false;
143 +}
144 +
145 +static void recover_database(const char *sqlite_database, const char *new_sqlite_database)
146 +{
147 + sqlite3 *database;
148 + int rc = sqlite3_open(sqlite_database, &database);
149 + if (rc != SQLITE_OK)
150 + return;
151 +
152 + netdata_log_info("Recover %s", sqlite_database);
153 + netdata_log_info(" to %s", new_sqlite_database);
154 +
155 + // This will remove the -shm and -wal files when we close the database
156 + db_execute(database, "select count(*) from sqlite_master limit 0");
157 +
158 + sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database);
159 + if (recover) {
160 +
161 + rc = sqlite3_recover_run(recover);
162 +
163 + if (rc == SQLITE_OK)
164 + netdata_log_info("Recover complete");
165 + else
166 + netdata_log_info("Recover encountered an error but the database may be usable");
167 +
168 + rc = sqlite3_recover_finish(recover);
169 +
170 + (void) sqlite3_close(database);
171 +
172 + if (rc == SQLITE_OK) {
173 + rc = rename(new_sqlite_database, sqlite_database);
174 + if (rc == 0) {
175 + netdata_log_info("Renamed %s", new_sqlite_database);
176 + netdata_log_info(" to %s", sqlite_database);
177 + }
178 + }
179 + }
180 + else
181 + (void) sqlite3_close(database);
182 +}
183 +
184 int execute_insert(sqlite3_stmt *res)
185 {
186 int rc;
@@ -130,6 +191,8 @@ int execute_insert(sqlite3_stmt *res)
191 error_report("Failed to insert/update, rc = %d -- attempt %d", rc, cnt);
192 }
193 else {
194 + if (rc == SQLITE_CORRUPT)
195 + (void) mark_database_to_recover(res, NULL);
196 error_report("SQLite error %d", rc);
197 break;
198 }
@@ -202,118 +265,7 @@ int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **state
265 return rc;
266 }
267
205 -static int check_table_integrity_cb(void *data, int argc, char **argv, char **column)
206 -{
207 - int *status = data;
208 - UNUSED(argc);
209 - UNUSED(column);
210 - netdata_log_info("---> %s", argv[0]);
211 - *status = (strcmp(argv[0], "ok") != 0);
212 - return 0;
213 -}
214 -
215 -
216 -static int check_table_integrity(char *table)
217 -{
218 - int status = 0;
219 - char *err_msg = NULL;
220 - char wstr[255];
221 -
222 - if (table) {
223 - netdata_log_info("Checking table %s", table);
224 - snprintfz(wstr, 254, "PRAGMA integrity_check(%s);", table);
225 - }
226 - else {
227 - netdata_log_info("Checking entire database");
228 - strcpy(wstr,"PRAGMA integrity_check;");
229 - }
230 -
231 - int rc = sqlite3_exec_monitored(db_meta, wstr, check_table_integrity_cb, (void *) &status, &err_msg);
232 - if (rc != SQLITE_OK) {
233 - error_report("SQLite error during database integrity check for %s, rc = %d (%s)",
234 - table ? table : "the entire database", rc, err_msg);
235 - sqlite3_free(err_msg);
236 - }
237 -
238 - return status;
239 -}
240 -
241 -const char *rebuild_chart_commands[] = {
242 - "BEGIN TRANSACTION; ",
243 - "DROP INDEX IF EXISTS ind_c1;" ,
244 - "DROP TABLE IF EXISTS chart_backup; " ,
245 - "CREATE TABLE chart_backup AS SELECT * FROM chart; " ,
246 - "DROP TABLE chart; ",
247 - "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, "
248 - "name text, family text, context text, title text, unit text, plugin text, "
249 - "module text, priority int, update_every int, chart_type int, memory_mode int, history_entries); ",
250 - "INSERT INTO chart SELECT DISTINCT * FROM chart_backup; ",
251 - "DROP TABLE chart_backup; " ,
252 - "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);",
253 - "COMMIT TRANSACTION;",
254 - NULL
255 -};
256 -
257 -static void rebuild_chart()
258 -{
259 - int rc;
260 - char *err_msg = NULL;
261 - netdata_log_info("Rebuilding chart table");
262 - for (int i = 0; rebuild_chart_commands[i]; i++) {
263 - netdata_log_info("Executing %s", rebuild_chart_commands[i]);
264 - rc = sqlite3_exec_monitored(db_meta, rebuild_chart_commands[i], 0, 0, &err_msg);
265 - if (rc != SQLITE_OK) {
266 - error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);
267 - error_report("SQLite failed statement %s", rebuild_chart_commands[i]);
268 - sqlite3_free(err_msg);
269 - }
270 - }
271 -}
272 -
273 -const char *rebuild_dimension_commands[] = {
274 - "BEGIN TRANSACTION; ",
275 - "DROP INDEX IF EXISTS ind_d1;" ,
276 - "DROP TABLE IF EXISTS dimension_backup; " ,
277 - "CREATE TABLE dimension_backup AS SELECT * FROM dimension; " ,
278 - "DROP TABLE dimension; " ,
279 - "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "
280 - "multiplier int, divisor int , algorithm int, options text);" ,
281 - "INSERT INTO dimension SELECT distinct * FROM dimension_backup; " ,
282 - "DROP TABLE dimension_backup; " ,
283 - "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);",
284 - "COMMIT TRANSACTION;",
285 - NULL
286 -};
287 -
288 -void rebuild_dimension()
289 -{
290 - int rc;
291 - char *err_msg = NULL;
292 -
293 - netdata_log_info("Rebuilding dimension table");
294 - for (int i = 0; rebuild_dimension_commands[i]; i++) {
295 - netdata_log_info("Executing %s", rebuild_dimension_commands[i]);
296 - rc = sqlite3_exec_monitored(db_meta, rebuild_dimension_commands[i], 0, 0, &err_msg);
297 - if (rc != SQLITE_OK) {
298 - error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);
299 - error_report("SQLite failed statement %s", rebuild_dimension_commands[i]);
300 - sqlite3_free(err_msg);
301 - }
302 - }
303 -}
304 -
305 -static int attempt_database_fix()
306 -{
307 - netdata_log_info("Closing database and attempting to fix it");
308 - int rc = sqlite3_close(db_meta);
309 - if (rc != SQLITE_OK)
310 - error_report("Failed to close database, rc = %d", rc);
311 - netdata_log_info("Attempting to fix database");
312 - db_meta = NULL;
313 - return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT, 0);
314 -}
315 -
316 -int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[])
268 +int init_database_batch(sqlite3 *database, const char *batch[])
269 {
270 int rc;
271 char *err_msg = NULL;
@@ -321,16 +273,14 @@ int init_database_batch(sqlite3 *database, int rebuild, int init_type, const cha
273 netdata_log_debug(D_METADATALOG, "Executing %s", batch[i]);
274 rc = sqlite3_exec_monitored(database, batch[i], 0, 0, &err_msg);
275 if (rc != SQLITE_OK) {
324 - error_report("SQLite error during database %s, rc = %d (%s)", init_type ? "cleanup" : "setup", rc, err_msg);
276 + error_report("SQLite error during database initialization, rc = %d (%s)", rc, err_msg);
277 error_report("SQLite failed statement %s", batch[i]);
278 analytics_set_data_str(&analytics_data.netdata_fail_reason, err_msg);
279 sqlite3_free(err_msg);
280 if (SQLITE_CORRUPT == rc) {
329 - if (!rebuild)
330 - return attempt_database_fix();
331 - rc = check_table_integrity(NULL);
332 - if (rc)
333 - error_report("Databse integrity errors reported");
281 + if (mark_database_to_recover(NULL, database))
282 + error_report("Database is corrupted will attempt to fix");
283 + return SQLITE_CORRUPT;
284 }
285 return 1;
286 }
@@ -390,8 +340,19 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
340 char sqlite_database[FILENAME_MAX + 1];
341 int rc;
342
393 - if (likely(!memory))
343 + if (likely(!memory)) {
344 + snprintfz(sqlite_database, FILENAME_MAX, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);
345 + rc = unlink(sqlite_database);
346 snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
347 +
348 + if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) {
349 + char new_sqlite_database[FILENAME_MAX + 1];
350 + snprintfz(new_sqlite_database, FILENAME_MAX, "%s/netdata-meta-recover.db", netdata_configured_cache_dir);
351 + recover_database(sqlite_database, new_sqlite_database);
352 + if (rebuild & DB_CHECK_RECOVER)
353 + return 0;
354 + }
355 + }
356 else
357 strcpy(sqlite_database, ":memory:");
358
@@ -404,45 +365,19 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
365 return 1;
366 }
367
407 - if (rebuild & (DB_CHECK_INTEGRITY | DB_CHECK_FIX_DB)) {
408 - int errors_detected = 0;
409 - if (!(rebuild & DB_CHECK_CONT))
410 - netdata_log_info("Running database check on %s", sqlite_database);
411 -
412 - if (check_table_integrity("chart")) {
413 - errors_detected++;
414 - if (rebuild & DB_CHECK_FIX_DB)
415 - rebuild_chart();
416 - else
417 - error_report("Errors reported -- run with -W sqlite-fix");
418 - }
419 -
420 - if (check_table_integrity("dimension")) {
421 - errors_detected++;
422 - if (rebuild & DB_CHECK_FIX_DB)
423 - rebuild_dimension();
424 - else
425 - error_report("Errors reported -- run with -W sqlite-fix");
426 - }
427 -
428 - if (!errors_detected) {
429 - if (check_table_integrity(NULL))
430 - error_report("Errors reported");
431 - }
432 - }
433 -
368 if (rebuild & DB_CHECK_RECLAIM_SPACE) {
435 - if (!(rebuild & DB_CHECK_CONT))
436 - netdata_log_info("Reclaiming space of %s", sqlite_database);
369 + netdata_log_info("Reclaiming space of %s", sqlite_database);
370 rc = sqlite3_exec_monitored(db_meta, "VACUUM;", 0, 0, &err_msg);
371 if (rc != SQLITE_OK) {
372 error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);
373 sqlite3_free(err_msg);
374 }
442 - }
443 -
444 - if (rebuild && !(rebuild & DB_CHECK_CONT))
375 + else {
376 + db_execute(db_meta, "select count(*) from sqlite_master limit 0");
377 + (void) sqlite3_close(db_meta);
378 + }
379 return 1;
380 + }
381
382 netdata_log_info("SQLite database %s initialization", sqlite_database);
383
@@ -469,41 +404,41 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
404 // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
405 // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
406 snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
472 - if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
407 + if(init_database_batch(db_meta, list)) return 1;
408
409 // https://www.sqlite.org/pragma.html#pragma_synchronous
410 // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
411 snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
477 - if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
412 + if(init_database_batch(db_meta, list)) return 1;
413
414 // https://www.sqlite.org/pragma.html#pragma_journal_mode
415 // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
416 snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
482 - if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
417 + if(init_database_batch(db_meta, list)) return 1;
418
419 // https://www.sqlite.org/pragma.html#pragma_temp_store
420 // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
421 snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
487 - if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
422 + if(init_database_batch(db_meta, list)) return 1;
423
424 // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
425 // PRAGMA schema.journal_size_limit = N ;
426 snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
492 - if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
427 + if(init_database_batch(db_meta, list)) return 1;
428
429 // https://www.sqlite.org/pragma.html#pragma_cache_size
430 // PRAGMA schema.cache_size = pages;
431 // PRAGMA schema.cache_size = -kibibytes;
432 snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
498 - if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
433 + if(init_database_batch(db_meta, list)) return 1;
434
435 snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
501 - if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
436 + if(init_database_batch(db_meta, list)) return 1;
437
503 - if (init_database_batch(db_meta, rebuild, 0, &database_config[0]))
438 + if (init_database_batch(db_meta, &database_config[0]))
439 return 1;
440
506 - if (init_database_batch(db_meta, rebuild, 0, &database_cleanup[0]))
441 + if (init_database_batch(db_meta, &database_cleanup[0]))
442 return 1;
443
444 netdata_log_info("SQLite database initialization completed");
database/sqlite/sqlite_functions.h
+5 -6
@@ -19,11 +19,10 @@ struct node_instance_list {
19 };
20
21 typedef enum db_check_action_type {
22 - DB_CHECK_NONE = 0x0000,
23 - DB_CHECK_INTEGRITY = 0x0001,
24 - DB_CHECK_FIX_DB = 0x0002,
25 - DB_CHECK_RECLAIM_SPACE = 0x0004,
26 - DB_CHECK_CONT = 0x00008
22 + DB_CHECK_NONE = (1 << 0),
23 + DB_CHECK_RECLAIM_SPACE = (1 << 1),
24 + DB_CHECK_CONT = (1 << 2),
25 + DB_CHECK_RECOVER = (1 << 3),
26 } db_check_action_type_t;
27
28 #define SQL_MAX_RETRY (100)
@@ -48,7 +47,7 @@ SQLITE_API int sqlite3_exec_monitored(
47 );
48
49 // Initialization and shutdown
51 -int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[]);
50 +int init_database_batch(sqlite3 *database, const char *batch[]);
51 int sql_init_database(db_check_action_type_t rebuild, int memory);
52 void sql_close_database(void);
53