@matheusbsbs / matheus-bagliano / commits / 50c725d6b6

fsmonitor--daemon: periodically truncate list of modified files

Teach fsmonitor--daemon to periodically truncate the list of modified files to save some memory. Clients will ask for the set of changes relative to a token that they found in the FSMN index extension in the index. (This token is like a point in time, but different). Clients will then update the index to contain the response token (so that subsequent commands will be relative to this new token). Therefore, the daemon can gradually truncate the in-memory list of changed paths as they become obsolete (older than the previous token). Since we may have multiple clients making concurrent requests with a skew of tokens and clients may be racing to the talk to the daemon, we lazily truncate the list. We introduce a 5 minute delay and truncate batches 5 minutes after they are considered obsolete. Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jeff Hostetler committed Mar 25, 2022 at 18:03 UTC 50c725d6b6f52b1c7fc4940c092d7f205cb99583

1 file changed +88

builtin/fsmonitor--daemon.c

+88

@@ -312,6 +312,75 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest,

312 batch_src->interned_paths[k];

313 }

314

315 +/*

316 + * To keep the batch list from growing unbounded in response to filesystem

317 + * activity, we try to truncate old batches from the end of the list as

318 + * they become irrelevant.

319 + *

320 + * We assume that the .git/index will be updated with the most recent token

321 + * any time the index is updated. And future commands will only ask for

322 + * recent changes *since* that new token. So as tokens advance into the

323 + * future, older batch items will never be requested/needed. So we can

324 + * truncate them without loss of functionality.

325 + *

326 + * However, multiple commands may be talking to the daemon concurrently

327 + * or perform a slow command, so a little "token skew" is possible.

328 + * Therefore, we want this to be a little bit lazy and have a generous

329 + * delay.

330 + *

331 + * The current reader thread walked backwards in time from `token->batch_head`

332 + * back to `batch_marker` somewhere in the middle of the batch list.

333 + *

334 + * Let's walk backwards in time from that marker an arbitrary delay

335 + * and truncate the list there. Note that these timestamps are completely

336 + * artificial (based on when we pinned the batch item) and not on any

337 + * filesystem activity.

338 + *

339 + * Return the obsolete portion of the list after we have removed it from

340 + * the official list so that the caller can free it after leaving the lock.

341 + */

342 +#define MY_TIME_DELAY_SECONDS (5 * 60) /* seconds */

343 +

344 +static struct fsmonitor_batch *with_lock__truncate_old_batches(

345 + struct fsmonitor_daemon_state *state,

346 + const struct fsmonitor_batch *batch_marker)

347 +{

348 + /* assert current thread holding state->main_lock */

349 +

350 + const struct fsmonitor_batch *batch;

351 + struct fsmonitor_batch *remainder;

352 +

353 + if (!batch_marker)

354 + return NULL;

355 +

356 + trace_printf_key(&trace_fsmonitor, "Truncate: mark (%"PRIu64",%"PRIu64")",

357 + batch_marker->batch_seq_nr,

358 + (uint64_t)batch_marker->pinned_time);

359 +

360 + for (batch = batch_marker; batch; batch = batch->next) {

361 + time_t t;

362 +

363 + if (!batch->pinned_time) /* an overflow batch */

364 + continue;

365 +

366 + t = batch->pinned_time + MY_TIME_DELAY_SECONDS;

367 + if (t > batch_marker->pinned_time) /* too close to marker */

368 + continue;

369 +

370 + goto truncate_past_here;

371 + }

372 +

373 + return NULL;

374 +

375 +truncate_past_here:

376 + state->current_token_data->batch_tail = (struct fsmonitor_batch *)batch;

377 +

378 + remainder = ((struct fsmonitor_batch *)batch)->next;

379 + ((struct fsmonitor_batch *)batch)->next = NULL;

380 +

381 + return remainder;

382 +}

383 +

384 static void fsmonitor_free_token_data(struct fsmonitor_token_data *token)

385 {

386 if (!token)

@@ -425,6 +494,7 @@ static int do_handle_client(struct fsmonitor_daemon_state *state,

494 const char *p;

495 const struct fsmonitor_batch *batch_head;

496 const struct fsmonitor_batch *batch;

497 + struct fsmonitor_batch *remainder = NULL;

498 intmax_t count = 0, duplicates = 0;

499 kh_str_t *shown;

500 int hash_ret;

@@ -652,11 +722,29 @@ static int do_handle_client(struct fsmonitor_daemon_state *state,

722 * that work.

723 */

724 fsmonitor_free_token_data(token_data);

725 + } else if (batch) {

726 + /*

727 + * We are holding the lock and are the only

728 + * reader of the ref-counted portion of the

729 + * list, so we get the honor of seeing if the

730 + * list can be truncated to save memory.

731 + *

732 + * The main loop did not walk to the end of the

733 + * list, so this batch is the first item in the

734 + * batch-list that is older than the requested

735 + * end-point sequence number. See if the tail

736 + * end of the list is obsolete.

737 + */

738 + remainder = with_lock__truncate_old_batches(state,

739 + batch);

740 }

741 }

742

743 pthread_mutex_unlock(&state->main_lock);

744

745 + if (remainder)

746 + fsmonitor_batch__free_list(remainder);

747 +

748 trace2_data_intmax("fsmonitor", the_repository, "response/length", total_response_len);

749 trace2_data_intmax("fsmonitor", the_repository, "response/count/files", count);

750 trace2_data_intmax("fsmonitor", the_repository, "response/count/duplicates", duplicates);