@cryptotaxi247 / netdata-1 / commits / 1368fdfc0

SimpleService and Charts optimization

lgz committed Nov 3, 2017 at 00:22 UTC 1368fdfc0097defc704b3953c47d17ec48e3ac16

2 files changed +85 -92

python.d/python_modules/bases/FrameworkServices/SimpleService.py

+14 -40

@@ -149,10 +149,6 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec

149 del self.order

150 del self.definitions

151

152 - # push charts to netdata

153 - for chart in self.charts:

154 - safe_print(chart.create())

155 -

152 # True if job has at least 1 chart else False

153 return bool(self.charts)

154

@@ -209,55 +205,33 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec

205 self.debug('get_data() returned incorrect type data')

206 return False

207

212 - charts_updated = False

213 -

214 - for chart in self.charts.penalty_exceeded(penalty_max=CHART_OBSOLETE_PENALTY):

215 - safe_print(chart.obsolete())

216 - chart.suppress()

217 - self.error("chart '{0}' was removed due to non updating".format(chart.name))

208 + updated = False

209

210 for chart in self.charts:

220 - if not chart.alive:

221 - continue

222 - dimension_updated, variables_updated = str(), str()

211

224 - for dimension in chart:

225 - try:

226 - value = int(data[dimension.id])

227 - except (KeyError, TypeError):

228 - continue

229 - else:

230 - dimension_updated += dimension.set(value)

212 + if chart.flags.obsolete:

213 + continue

214 + elif chart.penalty > CHART_OBSOLETE_PENALTY:

215 + chart.push_obsolete()

216 + self.error("chart '{0}' was removed due to non updating".format(chart.name))

217 + continue

218

232 - for var in chart.variables:

233 - try:

234 - value = int(data[var.id])

235 - except (KeyError, TypeError):

236 - continue

237 - else:

238 - variables_updated += var.set(value)

239 -

240 - if dimension_updated:

241 - charts_updated = True

242 - safe_print(''.join([chart.begin(since_last=interval),

243 - dimension_updated,

244 - variables_updated,

245 - 'END\n']))

246 - else:

247 - chart.penalty += 1

219 + since_last = 0 if chart.flags.new else interval

220 + ok = chart.update(data, since_last)

221 + if ok:

222 + updated = True

223

249 - if not charts_updated:

224 + if not updated:

225 self.debug('none of the charts have been updated')

226

252 - return charts_updated

227 + return updated

228

229 def manage_retries(self):

230 self._runtime_counters.RETRIES += 1

231 if self._runtime_counters.RETRIES % 5 == 0:

232 self._runtime_counters.PENALTY = int(self._runtime_counters.RETRIES * self.update_every / 2)

233 if self._runtime_counters.RETRIES >= self._runtime_counters.RETRIES_MAX:

259 - self.error('stopped after {retries_max} data '

260 - 'collection failures in a row'.format(retries_max=self._runtime_counters.RETRIES_MAX))

234 + self.error('stopped after {0} data collection failures in a row'.format(self._runtime_counters.RETRIES_MAX))

235 return False

236 return True

237

python.d/python_modules/bases/charts.py

+71 -52

@@ -109,13 +109,6 @@ class Charts:

109 def __nonzero__(self):

110 return self.__bool__()

111

112 - def penalty_exceeded(self, penalty_max):

113 - """

114 - :param penalty_max: <int>

115 - :return:

116 - """

117 - return (chart for chart in self if chart.penalty > penalty_max and chart.alive)

118 -

112 def add_chart(self, params):

113 """

114 Create Chart instance and add it to the dict

@@ -125,16 +118,16 @@ class Charts:

118 :return:

119 """

120 params = [self.job_name()] + params

128 - chart_id = params[1]

129 - if chart_id in self.charts:

130 - raise DuplicateItemError("'{chart}' already in charts".format(chart=chart_id))

131 - else:

132 - new_chart = Chart(params)

133 - new_chart.params['update_every'] = self.get_update_every()

134 - new_chart.params['priority'] = self.priority

135 - self.priority += 1

136 - self.charts[new_chart.id] = new_chart

137 - return new_chart

121 + new_chart = Chart(params)

122 + new_chart.params['update_every'] = self.get_update_every()

123 + new_chart.params['priority'] = self.priority

124 + self.priority += 1

125 + self.charts[new_chart.id] = new_chart

126 +

127 + return new_chart

128 +

129 + def active_charts(self):

130 + return [chart.id for chart in self if not chart.flags.obsolete]

131

132

133 class Chart:

@@ -156,7 +149,7 @@ class Chart:

149

150 self.dimensions = list()

151 self.variables = set()

159 - self.alive = True

152 + self.flags = ChartFlags()

153 self.penalty = 0

154

155 def __getattr__(self, item):

@@ -178,13 +171,6 @@ class Chart:

171 def __contains__(self, item):

172 return item in [dimension.id for dimension in self.dimensions]

173

181 - def suppress(self):

182 - self.alive = False

183 -

184 - def unsuppress(self):

185 - self.penalty = 0

186 - self.alive = True

187 -

174 def add_variable(self, variable):

175 """

176 :param variable: <list>

@@ -202,43 +188,69 @@ class Chart:

188 if dim.id in self:

189 raise DuplicateItemError("'{dimension}' already in '{chart}' dimensions".format(dimension=dim.id,

190 chart=self.name))

191 + self.refresh()

192 self.dimensions.append(dim)

193 return dim

194

208 - def add_dimension_and_push_chart(self, dimension):

209 - """

210 - :param dimension: <list>

211 - :return:

212 - """

213 - dim = self.add_dimension(dimension)

214 - self.unsuppress()

215 - safe_print(self.create(dim))

216 -

217 - def create(self, dimension=None):

195 + def create(self):

196 """

219 - :param dimension: Dimension

197 :return:

198 """

199 chart = CHART_CREATE.format(**self.params)

223 - if not dimension:

224 - dimensions = ''.join([dimension.create() for dimension in self.dimensions])

225 - variables = ''.join([var.set(var.value) for var in self.variables if var])

226 - return chart + dimensions + variables

200 + dimensions = ''.join([dimension.create() for dimension in self.dimensions])

201 + variables = ''.join([var.set(var.value) for var in self.variables if var])

202 +

203 + return chart + dimensions + variables

204 +

205 + def push_obsolete(self):

206 + self.flags.obsolete = True

207 + safe_print(CHART_OBSOLETE.format(**self.params))

208 +

209 + def update(self, data, since_last):

210 + updated_dimensions, updated_variables = str(), str()

211 +

212 + for dimension in self.dimensions:

213 + try:

214 + value = int(data[dimension.id])

215 + except (KeyError, TypeError):

216 + continue

217 + else:

218 + updated_dimensions += dimension.set(value)

219 +

220 + for var in self.variables:

221 + try:

222 + value = int(data[var.id])

223 + except (KeyError, TypeError):

224 + continue

225 + else:

226 + updated_variables += var.set(value)

227 +

228 + if updated_dimensions:

229 + if self.flags.push:

230 + self.push_created()

231 +

232 + chart_begin = CHART_BEGIN.format(type=self.type, id=self.id, since_last=since_last)

233 + self.push_updates(chart_begin, updated_dimensions, updated_variables, 'END\n')

234 else:

228 - dimensions = dimension.create()

229 - return chart + dimensions

235 + self.penalty += 1

236

231 - def begin(self, since_last):

232 - """

233 - :param since_last: <int>: microseconds

234 - :return:

235 - """

236 - return CHART_BEGIN.format(type=self.type,

237 - id=self.id,

238 - since_last=since_last)

237 + return bool(updated_dimensions)

238 +

239 + def push_created(self):

240 + self.penalty = 0

241 + self.flags.push = False

242 + self.flags.new = False

243 + safe_print(self.create())

244

240 - def obsolete(self):

241 - return CHART_OBSOLETE.format(**self.params)

245 + @staticmethod

246 + def push_updates(*data):

247 + safe_print(''.join(data))

248 +

249 + def refresh(self):

250 + self.flags.push = True

251 + if self.flags.obsolete:

252 + self.flags.new = True

253 + self.flags.obsolete = False

254

255

256 class Dimension:

@@ -332,3 +344,10 @@ class ChartVariable:

344 def set(self, value):

345 return CHART_VARIABLE_SET.format(id=self.id,

346 value=value)

347 +

348 +

349 +class ChartFlags:

350 + def __init__(self):

351 + self.new = True

352 + self.obsolete = False

353 + self.push = True