Commit | Line | Data |
---|---|---|
096102bd DG |
1 | /* |
2 | * Copyright (C) 2011 - David Goulet <dgoulet@efficios.com> | |
fbb9748b | 3 | * 2014 - Jan Glauber <jan.glauber@gmail.com> |
096102bd | 4 | * |
d14d33bf AM |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License, version 2 only, | |
7 | * as published by the Free Software Foundation. | |
096102bd | 8 | * |
d14d33bf AM |
9 | * This program is distributed in the hope that it will be useful, |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
096102bd | 13 | * |
d14d33bf AM |
14 | * You should have received a copy of the GNU General Public License along |
15 | * with this program; if not, write to the Free Software Foundation, Inc., | |
16 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
096102bd DG |
17 | */ |
18 | ||
d11b2027 MJ |
19 | /** |
20 | * @file modprobe.c | |
21 | * | |
22 | * @brief modprobe related functions. | |
23 | * | |
24 | */ | |
25 | ||
6c1c0768 | 26 | #define _LGPL_SOURCE |
c9d42407 | 27 | #include <assert.h> |
096102bd DG |
28 | #include <stdio.h> |
29 | #include <stdlib.h> | |
30 | #include <sys/wait.h> | |
31 | ||
32 | #include <common/common.h> | |
fbb9748b | 33 | #include <common/utils.h> |
096102bd DG |
34 | |
35 | #include "modprobe.h" | |
36 | #include "kern-modules.h" | |
37 | ||
ab57d7d3 JG |
38 | #define LTTNG_MOD_REQUIRED 1 |
39 | #define LTTNG_MOD_OPTIONAL 0 | |
40 | ||
41 | /* LTTng kernel tracer mandatory core modules list */ | |
42 | struct kern_modules_param kern_modules_control_core[] = { | |
ab57d7d3 JG |
43 | { "lttng-ring-buffer-client-discard" }, |
44 | { "lttng-ring-buffer-client-overwrite" }, | |
45 | { "lttng-ring-buffer-metadata-client" }, | |
46 | { "lttng-ring-buffer-client-mmap-discard" }, | |
47 | { "lttng-ring-buffer-client-mmap-overwrite" }, | |
48 | { "lttng-ring-buffer-metadata-mmap-client" }, | |
49 | }; | |
50 | ||
3fa9646c | 51 | /* LTTng kernel tracer probe modules list */ |
fbb9748b | 52 | struct kern_modules_param kern_modules_probes_default[] = { |
ab57d7d3 JG |
53 | { "lttng-probe-asoc" }, |
54 | { "lttng-probe-block" }, | |
55 | { "lttng-probe-btrfs" }, | |
56 | { "lttng-probe-compaction" }, | |
57 | { "lttng-probe-ext3" }, | |
58 | { "lttng-probe-ext4" }, | |
59 | { "lttng-probe-gpio" }, | |
60 | { "lttng-probe-irq" }, | |
61 | { "lttng-probe-jbd" }, | |
62 | { "lttng-probe-jbd2" }, | |
63 | { "lttng-probe-kmem" }, | |
64 | { "lttng-probe-kvm" }, | |
65 | { "lttng-probe-kvm-x86" }, | |
66 | { "lttng-probe-kvm-x86-mmu" }, | |
67 | { "lttng-probe-lock" }, | |
68 | { "lttng-probe-module" }, | |
69 | { "lttng-probe-napi" }, | |
70 | { "lttng-probe-net" }, | |
71 | { "lttng-probe-power" }, | |
72 | { "lttng-probe-printk" }, | |
73 | { "lttng-probe-random" }, | |
74 | { "lttng-probe-rcu" }, | |
75 | { "lttng-probe-regmap" }, | |
76 | { "lttng-probe-regulator" }, | |
77 | { "lttng-probe-rpm" }, | |
78 | { "lttng-probe-sched" }, | |
79 | { "lttng-probe-scsi" }, | |
80 | { "lttng-probe-signal" }, | |
81 | { "lttng-probe-skb" }, | |
82 | { "lttng-probe-sock" }, | |
83 | { "lttng-probe-statedump" }, | |
84 | { "lttng-probe-sunrpc" }, | |
85 | { "lttng-probe-timer" }, | |
86 | { "lttng-probe-udp" }, | |
87 | { "lttng-probe-vmscan" }, | |
88 | { "lttng-probe-v4l2" }, | |
89 | { "lttng-probe-workqueue" }, | |
90 | { "lttng-probe-writeback" }, | |
c66dfab8 | 91 | { "lttng-probe-x86-irq-vectors" }, |
037e00be | 92 | { "lttng-probe-x86-exceptions" }, |
096102bd DG |
93 | }; |
94 | ||
fbb9748b JG |
95 | /* dynamic probe modules list */ |
96 | static struct kern_modules_param *probes; | |
97 | static int nr_probes; | |
c9d42407 | 98 | static int probes_capacity; |
fbb9748b | 99 | |
234170ac UTL |
100 | #if HAVE_KMOD |
101 | #include <libkmod.h> | |
866c17ce | 102 | |
d11b2027 MJ |
103 | /** |
104 | * @brief Logging function for libkmod integration. | |
105 | */ | |
234170ac UTL |
106 | static void log_kmod(void *data, int priority, const char *file, int line, |
107 | const char *fn, const char *format, va_list args) | |
108 | { | |
109 | char *str; | |
110 | ||
111 | if (vasprintf(&str, format, args) < 0) { | |
112 | return; | |
113 | } | |
114 | ||
115 | DBG("libkmod: %s", str); | |
116 | free(str); | |
117 | } | |
866c17ce | 118 | |
d11b2027 MJ |
119 | /** |
120 | * @brief Setup the libkmod context. | |
121 | * | |
122 | * Create the context, add a custom logging function and preload the | |
123 | * ressources for faster operation. | |
124 | * | |
125 | * @returns \c 0 on success | |
126 | * \c < 0 on error | |
127 | */ | |
866c17ce | 128 | static int setup_kmod_ctx(struct kmod_ctx **ctx) |
234170ac | 129 | { |
866c17ce | 130 | int ret = 0; |
234170ac | 131 | |
866c17ce | 132 | *ctx = kmod_new(NULL, NULL); |
234170ac UTL |
133 | if (!ctx) { |
134 | PERROR("Unable to create kmod library context"); | |
135 | ret = -ENOMEM; | |
136 | goto error; | |
137 | } | |
138 | ||
866c17ce MJ |
139 | kmod_set_log_fn(*ctx, log_kmod, NULL); |
140 | ret = kmod_load_resources(*ctx); | |
141 | if (ret < 0) { | |
142 | ERR("Failed to load kmod library resources"); | |
143 | goto error; | |
144 | } | |
145 | ||
146 | error: | |
147 | return ret; | |
148 | } | |
149 | ||
d11b2027 MJ |
150 | /** |
151 | * @brief Loads the kernel modules in \p modules | |
152 | * | |
153 | * @param modules List of modules to load | |
154 | * @param entries Number of modules in the list | |
155 | * @param required Are the modules required or optionnal | |
156 | * | |
157 | * If the modules are required, we will return with error after the | |
158 | * first failed module load, otherwise we continue loading. | |
159 | * | |
160 | * @returns \c 0 on success | |
161 | * \c < 0 on error | |
162 | */ | |
866c17ce MJ |
163 | static int modprobe_lttng(struct kern_modules_param *modules, |
164 | int entries, int required) | |
165 | { | |
166 | int ret = 0, i; | |
167 | struct kmod_ctx *ctx; | |
168 | ||
169 | ret = setup_kmod_ctx(&ctx); | |
170 | if (ret < 0) { | |
171 | goto error; | |
172 | } | |
234170ac UTL |
173 | |
174 | for (i = 0; i < entries; i++) { | |
175 | struct kmod_module *mod = NULL; | |
176 | ||
177 | ret = kmod_module_new_from_name(ctx, modules[i].name, &mod); | |
178 | if (ret < 0) { | |
179 | PERROR("Failed to create kmod module for %s", modules[i].name); | |
180 | goto error; | |
181 | } | |
182 | ||
183 | ret = kmod_module_probe_insert_module(mod, KMOD_PROBE_IGNORE_LOADED, | |
184 | NULL, NULL, NULL, NULL); | |
16c2e854 PP |
185 | if (ret < 0) { |
186 | if (required) { | |
187 | ERR("Unable to load required module %s", | |
188 | modules[i].name); | |
189 | goto error; | |
190 | } else { | |
191 | DBG("Unable to load optional module %s; continuing", | |
192 | modules[i].name); | |
193 | ret = 0; | |
194 | } | |
234170ac UTL |
195 | } else { |
196 | DBG("Modprobe successfully %s", modules[i].name); | |
197 | } | |
198 | ||
199 | kmod_module_unref(mod); | |
200 | } | |
201 | ||
202 | error: | |
203 | if (ctx) { | |
204 | kmod_unref(ctx); | |
205 | } | |
206 | return ret; | |
207 | } | |
208 | ||
d11b2027 MJ |
209 | /** |
210 | * @brief Recursively unload modules. | |
211 | * | |
212 | * This function implements the same modules unloading behavior as | |
213 | * 'modprobe -r' or rmmod, it will recursevily go trought the \p module | |
214 | * dependencies and unload modules with a refcount of 0. | |
215 | * | |
216 | * @param mod The module to unload | |
217 | * | |
218 | * @returns \c 0 on success | |
219 | * \c < 0 on error | |
220 | */ | |
866c17ce MJ |
221 | static int rmmod_recurse(struct kmod_module *mod) { |
222 | int ret = 0; | |
223 | struct kmod_list *deps, *itr; | |
224 | ||
225 | if (kmod_module_get_initstate(mod) == KMOD_MODULE_BUILTIN) { | |
226 | DBG("Module %s is builtin", kmod_module_get_name(mod)); | |
227 | return ret; | |
228 | } | |
229 | ||
230 | ret = kmod_module_remove_module(mod, 0); | |
231 | ||
232 | deps = kmod_module_get_dependencies(mod); | |
233 | if (deps != NULL) { | |
234 | kmod_list_foreach(itr, deps) { | |
235 | struct kmod_module *dep = kmod_module_get_module(itr); | |
236 | if (kmod_module_get_refcnt(dep) == 0) { | |
237 | DBG("Recursive remove module %s", | |
238 | kmod_module_get_name(dep)); | |
239 | rmmod_recurse(dep); | |
240 | } | |
241 | kmod_module_unref(dep); | |
242 | } | |
243 | kmod_module_unref_list(deps); | |
244 | } | |
245 | ||
246 | return ret; | |
247 | } | |
248 | ||
d11b2027 MJ |
249 | /** |
250 | * @brief Unloads the kernel modules in \p modules | |
251 | * | |
252 | * @param modules List of modules to unload | |
253 | * @param entries Number of modules in the list | |
254 | * @param required Are the modules required or optionnal | |
255 | * | |
256 | */ | |
866c17ce MJ |
257 | static void modprobe_remove_lttng(const struct kern_modules_param *modules, |
258 | int entries, int required) | |
259 | { | |
260 | int ret = 0, i; | |
261 | struct kmod_ctx *ctx; | |
262 | ||
263 | ret = setup_kmod_ctx(&ctx); | |
264 | if (ret < 0) { | |
265 | goto error; | |
266 | } | |
267 | ||
268 | for (i = entries - 1; i >= 0; i--) { | |
269 | struct kmod_module *mod = NULL; | |
270 | ||
271 | ret = kmod_module_new_from_name(ctx, modules[i].name, &mod); | |
272 | if (ret < 0) { | |
273 | PERROR("Failed to create kmod module for %s", modules[i].name); | |
274 | goto error; | |
275 | } | |
276 | ||
277 | ret = rmmod_recurse(mod); | |
278 | if (ret == -EEXIST) { | |
279 | DBG("Module %s is not in kernel.", modules[i].name); | |
280 | } else if (required && ret < 0) { | |
281 | ERR("Unable to remove module %s", modules[i].name); | |
282 | } else { | |
283 | DBG("Modprobe removal successful %s", | |
284 | modules[i].name); | |
285 | } | |
286 | ||
287 | kmod_module_unref(mod); | |
288 | } | |
289 | ||
290 | error: | |
291 | if (ctx) { | |
292 | kmod_unref(ctx); | |
293 | } | |
294 | } | |
295 | ||
234170ac UTL |
296 | #else /* HAVE_KMOD */ |
297 | ||
fbb9748b | 298 | static int modprobe_lttng(struct kern_modules_param *modules, |
234170ac | 299 | int entries, int required) |
096102bd DG |
300 | { |
301 | int ret = 0, i; | |
302 | char modprobe[256]; | |
303 | ||
e23b81ed | 304 | for (i = 0; i < entries; i++) { |
096102bd DG |
305 | ret = snprintf(modprobe, sizeof(modprobe), |
306 | "/sbin/modprobe %s%s", | |
ab57d7d3 | 307 | required ? "" : "-q ", |
e23b81ed | 308 | modules[i].name); |
096102bd DG |
309 | if (ret < 0) { |
310 | PERROR("snprintf modprobe"); | |
311 | goto error; | |
312 | } | |
313 | modprobe[sizeof(modprobe) - 1] = '\0'; | |
314 | ret = system(modprobe); | |
315 | if (ret == -1) { | |
16c2e854 PP |
316 | if (required) { |
317 | ERR("Unable to launch modprobe for required module %s", | |
318 | modules[i].name); | |
319 | goto error; | |
320 | } else { | |
321 | DBG("Unable to launch modprobe for optional module %s; continuing", | |
322 | modules[i].name); | |
323 | ret = 0; | |
324 | } | |
325 | } else if (WEXITSTATUS(ret) != 0) { | |
326 | if (required) { | |
327 | ERR("Unable to load required module %s", | |
328 | modules[i].name); | |
329 | goto error; | |
330 | } else { | |
331 | DBG("Unable to load optional module %s; continuing", | |
332 | modules[i].name); | |
333 | ret = 0; | |
334 | } | |
096102bd | 335 | } else { |
ab57d7d3 | 336 | DBG("Modprobe successfully %s", modules[i].name); |
096102bd DG |
337 | } |
338 | } | |
339 | ||
340 | error: | |
341 | return ret; | |
342 | } | |
343 | ||
35e090b7 MJ |
344 | static void modprobe_remove_lttng(const struct kern_modules_param *modules, |
345 | int entries, int required) | |
346 | { | |
347 | int ret = 0, i; | |
348 | char modprobe[256]; | |
349 | ||
350 | for (i = entries - 1; i >= 0; i--) { | |
351 | ret = snprintf(modprobe, sizeof(modprobe), | |
352 | "/sbin/modprobe -r -q %s", | |
353 | modules[i].name); | |
354 | if (ret < 0) { | |
355 | PERROR("snprintf modprobe -r"); | |
356 | return; | |
357 | } | |
358 | modprobe[sizeof(modprobe) - 1] = '\0'; | |
359 | ret = system(modprobe); | |
360 | if (ret == -1) { | |
361 | ERR("Unable to launch modprobe -r for module %s", | |
362 | modules[i].name); | |
363 | } else if (required && WEXITSTATUS(ret) != 0) { | |
364 | ERR("Unable to remove module %s", | |
365 | modules[i].name); | |
366 | } else { | |
367 | DBG("Modprobe removal successful %s", | |
368 | modules[i].name); | |
369 | } | |
370 | } | |
371 | } | |
372 | ||
866c17ce MJ |
373 | #endif /* HAVE_KMOD */ |
374 | ||
35e090b7 MJ |
375 | /* |
376 | * Remove control kernel module(s) in reverse load order. | |
377 | */ | |
378 | void modprobe_remove_lttng_control(void) | |
379 | { | |
380 | modprobe_remove_lttng(kern_modules_control_core, | |
381 | ARRAY_SIZE(kern_modules_control_core), | |
382 | LTTNG_MOD_REQUIRED); | |
383 | } | |
384 | ||
385 | static void free_probes(void) | |
386 | { | |
387 | int i; | |
388 | ||
389 | if (!probes) { | |
390 | return; | |
391 | } | |
392 | for (i = 0; i < nr_probes; ++i) { | |
393 | free(probes[i].name); | |
394 | } | |
395 | free(probes); | |
396 | probes = NULL; | |
397 | nr_probes = 0; | |
398 | } | |
399 | ||
400 | /* | |
401 | * Remove data kernel modules in reverse load order. | |
402 | */ | |
403 | void modprobe_remove_lttng_data(void) | |
404 | { | |
405 | if (!probes) { | |
406 | return; | |
407 | } | |
408 | modprobe_remove_lttng(probes, nr_probes, LTTNG_MOD_OPTIONAL); | |
409 | free_probes(); | |
410 | } | |
411 | ||
412 | /* | |
413 | * Remove all kernel modules in reverse order. | |
414 | */ | |
415 | void modprobe_remove_lttng_all(void) | |
416 | { | |
417 | modprobe_remove_lttng_data(); | |
418 | modprobe_remove_lttng_control(); | |
419 | } | |
420 | ||
e23b81ed JG |
421 | /* |
422 | * Load control kernel module(s). | |
423 | */ | |
424 | int modprobe_lttng_control(void) | |
425 | { | |
ab57d7d3 JG |
426 | int ret; |
427 | ||
428 | ret = modprobe_lttng(kern_modules_control_core, | |
429 | ARRAY_SIZE(kern_modules_control_core), | |
430 | LTTNG_MOD_REQUIRED); | |
ab57d7d3 | 431 | return ret; |
e23b81ed | 432 | } |
ab57d7d3 | 433 | |
c9d42407 PP |
434 | /** |
435 | * Grow global list of probes (double capacity or set it to 1 if | |
436 | * currently 0 and copy existing data). | |
096102bd | 437 | */ |
c9d42407 | 438 | static int grow_probes(void) |
096102bd | 439 | { |
c9d42407 PP |
440 | int i; |
441 | struct kern_modules_param *tmp_probes; | |
fbb9748b | 442 | |
c9d42407 PP |
443 | /* Initialize capacity to 1 if 0. */ |
444 | if (probes_capacity == 0) { | |
445 | probes = zmalloc(sizeof(*probes)); | |
446 | if (!probes) { | |
447 | PERROR("malloc probe list"); | |
448 | return -ENOMEM; | |
449 | } | |
450 | ||
451 | probes_capacity = 1; | |
452 | return 0; | |
fbb9748b JG |
453 | } |
454 | ||
c9d42407 PP |
455 | /* Double size. */ |
456 | probes_capacity *= 2; | |
457 | ||
458 | tmp_probes = zmalloc(sizeof(*tmp_probes) * probes_capacity); | |
459 | if (!tmp_probes) { | |
fbb9748b JG |
460 | PERROR("malloc probe list"); |
461 | return -ENOMEM; | |
462 | } | |
463 | ||
c9d42407 PP |
464 | for (i = 0; i < nr_probes; ++i) { |
465 | /* Move name pointer. */ | |
466 | tmp_probes[i].name = probes[i].name; | |
467 | } | |
468 | ||
469 | /* Replace probes with larger copy. */ | |
470 | free(probes); | |
471 | probes = tmp_probes; | |
472 | ||
473 | return 0; | |
474 | } | |
475 | ||
476 | /* | |
477 | * Appends a comma-separated list of probes to the global list | |
478 | * of probes. | |
479 | */ | |
480 | static int append_list_to_probes(const char *list) | |
481 | { | |
482 | char *next; | |
d3c04b7c | 483 | int ret; |
44603c80 | 484 | char *tmp_list, *cur_list; |
c9d42407 PP |
485 | |
486 | assert(list); | |
487 | ||
44603c80 | 488 | cur_list = tmp_list = strdup(list); |
c9d42407 PP |
489 | if (!tmp_list) { |
490 | PERROR("strdup temp list"); | |
491 | return -ENOMEM; | |
492 | } | |
493 | ||
494 | for (;;) { | |
fbb9748b | 495 | size_t name_len; |
c9d42407 | 496 | struct kern_modules_param *cur_mod; |
fbb9748b | 497 | |
44603c80 | 498 | next = strtok(cur_list, ","); |
fbb9748b | 499 | if (!next) { |
c9d42407 | 500 | break; |
fbb9748b | 501 | } |
44603c80 | 502 | cur_list = NULL; |
fbb9748b JG |
503 | |
504 | /* filter leading spaces */ | |
505 | while (*next == ' ') { | |
506 | next++; | |
507 | } | |
508 | ||
c9d42407 PP |
509 | if (probes_capacity <= nr_probes) { |
510 | ret = grow_probes(); | |
511 | if (ret) { | |
398d5459 | 512 | goto error; |
c9d42407 PP |
513 | } |
514 | } | |
515 | ||
fbb9748b JG |
516 | /* Length 13 is "lttng-probe-" + \0 */ |
517 | name_len = strlen(next) + 13; | |
518 | ||
d3c04b7c | 519 | cur_mod = &probes[nr_probes]; |
c9d42407 PP |
520 | cur_mod->name = zmalloc(name_len); |
521 | if (!cur_mod->name) { | |
fbb9748b | 522 | PERROR("malloc probe list"); |
398d5459 MD |
523 | ret = -ENOMEM; |
524 | goto error; | |
fbb9748b JG |
525 | } |
526 | ||
c9d42407 | 527 | ret = snprintf(cur_mod->name, name_len, "lttng-probe-%s", next); |
fbb9748b JG |
528 | if (ret < 0) { |
529 | PERROR("snprintf modprobe name"); | |
398d5459 MD |
530 | ret = -ENOMEM; |
531 | goto error; | |
fbb9748b | 532 | } |
c9d42407 | 533 | |
c9d42407 | 534 | nr_probes++; |
fbb9748b JG |
535 | } |
536 | ||
c9d42407 | 537 | free(tmp_list); |
c9d42407 | 538 | return 0; |
398d5459 MD |
539 | |
540 | error: | |
541 | free(tmp_list); | |
542 | free_probes(); | |
543 | return ret; | |
c9d42407 PP |
544 | } |
545 | ||
546 | /* | |
547 | * Load data kernel module(s). | |
548 | */ | |
549 | int modprobe_lttng_data(void) | |
550 | { | |
551 | int ret, i; | |
552 | char *list; | |
553 | ||
554 | /* | |
555 | * Base probes: either from command line option, environment | |
556 | * variable or default list. | |
557 | */ | |
558 | if (kmod_probes_list) { | |
559 | list = kmod_probes_list; | |
560 | } else { | |
561 | list = utils_get_kmod_probes_list(); | |
562 | } | |
563 | ||
564 | if (list) { | |
565 | /* User-specified probes. */ | |
566 | ret = append_list_to_probes(list); | |
c9d42407 PP |
567 | if (ret) { |
568 | return ret; | |
569 | } | |
570 | } else { | |
571 | /* Default probes. */ | |
572 | int def_len = ARRAY_SIZE(kern_modules_probes_default); | |
c9d42407 | 573 | |
62e0422e | 574 | probes = zmalloc(sizeof(*probes) * def_len); |
c9d42407 PP |
575 | if (!probes) { |
576 | PERROR("malloc probe list"); | |
577 | return -ENOMEM; | |
578 | } | |
579 | ||
580 | nr_probes = probes_capacity = def_len; | |
581 | ||
582 | for (i = 0; i < def_len; ++i) { | |
583 | char* name = strdup(kern_modules_probes_default[i].name); | |
584 | ||
585 | if (!name) { | |
586 | PERROR("strdup probe item"); | |
398d5459 MD |
587 | ret = -ENOMEM; |
588 | goto error; | |
c9d42407 PP |
589 | } |
590 | ||
591 | probes[i].name = name; | |
592 | } | |
593 | } | |
594 | ||
595 | /* | |
596 | * Extra modules? Append them to current probes list. | |
597 | */ | |
598 | if (kmod_extra_probes_list) { | |
599 | list = kmod_extra_probes_list; | |
600 | } else { | |
601 | list = utils_get_extra_kmod_probes_list(); | |
602 | } | |
603 | ||
604 | if (list) { | |
605 | ret = append_list_to_probes(list); | |
606 | if (ret) { | |
398d5459 | 607 | goto error; |
c9d42407 PP |
608 | } |
609 | } | |
610 | ||
611 | /* | |
612 | * Load probes modules now. | |
613 | */ | |
398d5459 MD |
614 | ret = modprobe_lttng(probes, nr_probes, LTTNG_MOD_OPTIONAL); |
615 | if (ret) { | |
616 | goto error; | |
617 | } | |
618 | return ret; | |
619 | ||
620 | error: | |
621 | free_probes(); | |
622 | return ret; | |
096102bd | 623 | } |