@cryptotaxi247 / netdata-1 / commits / 51cff0660

Improved error handling and recovery during compaction and metadata log replay (#9354)

* Improved error handling and recovery during compaction and metadata log replay

Stelios Fragkakis committed Jun 19, 2020 at 11:31 UTC 51cff0660b0eb671dbec42e54d7567db8b6ba712

7 files changed +98 -53

database/engine/metadata_log/compaction.c

+11 -3

@@ -87,7 +87,7 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)

87 ret = find_object_by_guid(uuid, NULL, 0);

88 switch (ret) {

89 case GUID_TYPE_CHAR:

90 - fatal_assert(0);

90 + error_with_guid(uuid, "Ignoring unexpected type GUID_TYPE_CHAR");

91 break;

92 case GUID_TYPE_CHART:

93 st = metalog_get_chart_from_uuid(ctx, uuid);

@@ -106,7 +106,12 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)

106 case GUID_TYPE_DIMENSION:

107 rd = metalog_get_dimension_from_uuid(ctx, uuid);

108 if (rd) {

109 - if (ctx->current_compaction_id > rd->state->compaction_id) {

109 + if (ctx->current_compaction_id > rd->rrdset->compaction_id) {

110 + error("Forcing compaction of chart %s", rd->rrdset->id);

111 + rd->rrdset->compaction_id = ctx->current_compaction_id;

112 + buffer = metalog_update_chart_buffer(rd->rrdset, ctx->current_compaction_id);

113 + metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, rd->rrdset->chart_uuid, 1);

114 + } else if (ctx->current_compaction_id > rd->state->compaction_id) {

115 rd->state->compaction_id = ctx->current_compaction_id;

116 buffer = metalog_update_dimension_buffer(rd);

117 metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);

@@ -129,8 +134,11 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)

134 case GUID_TYPE_NOTFOUND:

135 debug(D_METADATALOG, "Ignoring nonexistent metadata record.");

136 break;

137 + case GUID_TYPE_NOSPACE:

138 + error_with_guid(uuid, "Not enough space for object retrieval");

139 + break;

140 default:

133 - fatal_assert(0);

141 + error("Unknown return code %u from find_object_by_guid", ret);

142 break;

143 }

144 }

database/engine/metadata_log/logfile.c

-7

@@ -524,13 +524,6 @@ static void iterate_records(struct metadata_logfile *metalogfile)

524 pos);

525

526 replay_record(metalogfile, header, buf + header->header_length);

527 - if (!uuid_is_null(state->uuid)) { /* It's a valid object */

528 - struct metalog_record record;

529 -

530 - uuid_copy(record.uuid, state->uuid);

531 - mlf_record_insert(metalogfile, &record);

532 - uuid_clear(state->uuid); /* Clear state for parsing of next record */

533 - }

527 }

528

529 freez(buf);

database/engine/metadata_log/metadatalog.c

@@ -406,3 +406,12 @@ error_after_loop_init:

406 /* wake up initialization thread */

407 complete(&ctx->metalog_completion);

408 }

409 +

410 +void error_with_guid(uuid_t *uuid, char *reason)

411 +{

412 + char uuid_str[37];

413 +

414 + uuid_unparse_lower(*uuid, uuid_str);

415 + errno = 0;

416 + error("%s (GUID = %s)", reason, uuid_str);

417 +}

\ No newline at end of file

database/engine/metadata_log/metadatalog.h

+1 -1

@@ -132,5 +132,5 @@ extern void metalog_test_quota(struct metalog_worker_config *wc);

132 extern void metalog_worker(void* arg);

133 extern void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd);

134 extern struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc);

135 -

135 +extern void error_with_guid(uuid_t *uuid, char *reason);

136 #endif /* NETDATA_METADATALOG_H */

database/engine/metadata_log/metadatalogapi.c

+20 -8

@@ -273,16 +273,21 @@ RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_

273 uuid_t *machine_guid, *chart_char_guid;

274

275 ret = find_object_by_guid(chart_uuid, chart_object, 33);

276 - fatal_assert(GUID_TYPE_CHART == ret);

276 + if (unlikely(GUID_TYPE_CHART != ret))

277 + return NULL;

278

279 machine_guid = (uuid_t *)chart_object;

280 RRDHOST *host = ctx->rrdeng_ctx->host;

280 - fatal_assert(!uuid_compare(host->host_uuid, *machine_guid));

281 + if (unlikely(uuid_compare(host->host_uuid, *machine_guid))) {

282 + error("Metadata host machine GUID does not match the one assosiated with the chart");

283 + return NULL;

284 + }

285

286 chart_char_guid = (uuid_t *)(chart_object + 16);

287

288 ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);

285 - fatal_assert(GUID_TYPE_CHAR == ret);

289 + if (unlikely(GUID_TYPE_CHAR != ret))

290 + return NULL;

291 RRDSET *st = rrdset_find(host, chart_fullid);

292

293 return st;

@@ -300,22 +305,29 @@ RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *me

305

306 machine_guid = (uuid_t *)dim_object;

307 RRDHOST *host = ctx->rrdeng_ctx->host;

303 - fatal_assert(!uuid_compare(host->host_uuid, *machine_guid));

308 + if (unlikely(uuid_compare(host->host_uuid, *machine_guid))) {

309 + error("Metadata host machine GUID does not match the one assosiated with the dimension");

310 + return NULL;

311 + }

312

313 chart_guid = (uuid_t *)(dim_object + 16);

314 dim_char_guid = (uuid_t *)(dim_object + 16 + 16);

315

316 ret = find_object_by_guid(dim_char_guid, id_str, sizeof(id_str));

309 - fatal_assert(GUID_TYPE_CHAR == ret);

317 + if (unlikely(GUID_TYPE_CHAR != ret))

318 + return NULL;

319

320 ret = find_object_by_guid(chart_guid, chart_object, sizeof(chart_object));

312 - fatal_assert(GUID_TYPE_CHART == ret);

321 + if (unlikely(GUID_TYPE_CHART != ret))

322 + return NULL;

323 chart_char_guid = (uuid_t *)(chart_object + 16);

324

325 ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);

316 - fatal_assert(GUID_TYPE_CHAR == ret);

326 + if (unlikely(GUID_TYPE_CHAR != ret))

327 + return NULL;

328 RRDSET *st = rrdset_find(host, chart_fullid);

318 - fatal_assert(st);

329 + if (!st)

330 + return NULL;

331

332 RRDDIM *rd = rrddim_find(st, id_str);

333

database/engine/metadata_log/metalogpluginsd.c

+49 -30

@@ -116,41 +116,55 @@ PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)

116

117 ret = find_object_by_guid(uuid, object, 49);

118 switch (ret) {

119 - case GUID_TYPE_CHAR:

120 - fatal_assert(0);

121 - break;

122 - case GUID_TYPE_CHART:

123 - case GUID_TYPE_DIMENSION:

124 - host = ctx->rrdeng_ctx->host;

125 - switch (ret) {

126 - case GUID_TYPE_CHART:

127 - chart_char_guid = (uuid_t *)(object + 16);

128 -

129 - ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);

130 - fatal_assert(GUID_TYPE_CHAR == ret);

131 - ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);

119 + case GUID_TYPE_NOTFOUND:

120 + if (unlikely(ctx->rrdeng_ctx->host && uuid_compare(ctx->rrdeng_ctx->host->host_uuid, *uuid)))

121 + error_with_guid(uuid, "Failed to find valid context");

122 + break;

123 + case GUID_TYPE_CHAR:

124 + error_with_guid(uuid, "Ignoring unexpected type GUID_TYPE_CHAR");

125 break;

126 + case GUID_TYPE_CHART:

127 case GUID_TYPE_DIMENSION:

134 - chart_guid = (uuid_t *)(object + 16);

135 -

136 - ret = find_object_by_guid(chart_guid, chart_object, 33);

137 - fatal_assert(GUID_TYPE_CHART == ret);

138 - chart_char_guid = (uuid_t *)(chart_object + 16);

139 -

140 - ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);

141 - fatal_assert(GUID_TYPE_CHAR == ret);

142 - ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);

128 + host = ctx->rrdeng_ctx->host;

129 + switch (ret) {

130 + case GUID_TYPE_CHART:

131 + chart_char_guid = (uuid_t *)(object + 16);

132 +

133 + ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);

134 + if (unlikely(GUID_TYPE_CHAR != ret))

135 + error_with_guid(uuid, "Failed to find valid chart name");

136 + else

137 + ((PARSER_USER_OBJECT *)user)->st = rrdset_find(host, id_str);

138 + break;

139 + case GUID_TYPE_DIMENSION:

140 + chart_guid = (uuid_t *)(object + 16);

141 +

142 + ret = find_object_by_guid(chart_guid, chart_object, 33);

143 + if (unlikely(GUID_TYPE_CHART != ret)) {

144 + error_with_guid(uuid, "Failed to find valid chart");

145 + break;

146 + }

147 + chart_char_guid = (uuid_t *)(object + 16);

148 +

149 + ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);

150 + if (unlikely(GUID_TYPE_CHAR != ret))

151 + error_with_guid(uuid, "Failed to find valid chart name");

152 + else

153 + ((PARSER_USER_OBJECT *)user)->st = rrdset_find(host, id_str);

154 + break;

155 + default:

156 + break;

157 + }

158 + break;

159 + case GUID_TYPE_HOST:

160 + /* Ignore for now */

161 + break;

162 + case GUID_TYPE_NOSPACE:

163 + error_with_guid(uuid, "Not enough space for object retrieval");

164 break;

165 default:

145 - fatal_assert(0);

166 + error("Unknown return code %u from find_object_by_guid", ret);

167 break;

147 - }

148 - break;

149 - case GUID_TYPE_HOST:

150 - /* Ignore for now */

151 - break;

152 - default:

153 - break;

168 }

169

170 return PARSER_RC_OK;

@@ -176,6 +190,8 @@ PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)

190 rrdhost_wrlock(host);

191 rrdset_free(st);

192 rrdhost_unlock(host);

193 + } else {

194 + debug(D_METADATALOG, "Ignoring nonexistent chart metadata record.");

195 }

196 break;

197 case GUID_TYPE_DIMENSION:

@@ -186,6 +202,9 @@ PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)

202 rrddim_free_custom(st, rd, 0);

203 rrdset_unlock(st);

204 }

205 + else {

206 + debug(D_METADATALOG, "Ignoring nonexistent dimension metadata record.");

207 + }

208 break;

209 case GUID_TYPE_HOST:

210 /* Ignore for now */

database/engine/rrdengineapi.c

+8 -4

@@ -74,10 +74,14 @@ void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid)

74 if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION,

75 replace_instead_of_generate))) {

76 errno = 0;

77 - error("FAILED to generate GUID for %s", rd->id);

78 - freez(rd->state->metric_uuid);

79 - rd->state->metric_uuid = NULL;

80 - fatal_assert(0);

77 + error("FAILED to reuse GUID for %s", rd->id);

78 + if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION, 0))) {

79 + errno = 0;

80 + error("FAILED to generate GUID for %s", rd->id);

81 + freez(rd->state->metric_uuid);

82 + rd->state->metric_uuid = NULL;

83 + fatal_assert(0);

84 + }

85 }

87 uv_rwlock_rdlock(&pg_cache->metrics_index.lock);