В рандомные моменты Мантикор аллоцирует всю доступную память, уходит в своп и перестаёт обслужить запросы. Помогает только kill -9 searchd-процесса и запуск его.
Каких-то закономерностей этого поведения пока нам не удалось обнаружить.
Машина, на которой развёрнут мантикор:
128 ядер
64 GB RAM
swap 8 GB
под индексы используется 1TB NVME диск
индекс products 5.5GB занимает на диске
сеть 1Гбит/с
Записей в индексе: 2млн
DDL индекса:
CREATE TABLE products (
id bigint,
name text stored,
slug text stored,
description text stored,
selling_text text stored,
additional_images text stored,
school_umk_title text,
seo_description text stored,
author_full_name_list text,
publisher_series_name text,
publisher_name text,
publisher_brand_name text,
manufacturer_brand_name text,
image string attribute,
price_retail float,
marketing_status_list multi,
availability_status integer,
availability_quantity integer,
vendor_id integer,
nds integer,
reasons_to_buy_list json,
literature_work_publishing_year integer,
preorder_available_at timestamp,
released_at timestamp,
created_at timestamp,
updated_at timestamp,
isbns string attribute,
printing_page_count integer,
printing_page_format string attribute,
printing_copy_count integer,
weight integer,
height integer,
width integer,
length integer,
excerpts json,
rating_average float,
rating_weight integer,
rating_star integer,
rating_count integer,
review_count integer,
purchase_stats_day_avg_count integer,
sales_stat_year_turnover_amount_without_vat float,
sales_stat_year_turnover_quantity integer,
sales_stat_year_markup float,
sales_stat_half_year_turnover_amount_without_vat float,
sales_stat_half_year_turnover_quantity integer,
sales_stat_half_year_markup float,
sales_stat_quarter_turnover_amount_without_vat float,
sales_stat_quarter_turnover_quantity integer,
sales_stat_quarter_markup float,
sales_stat_month_turnover_amount_without_vat float,
sales_stat_month_turnover_quantity integer,
sales_stat_month_markup float,
school_grade_id_list multi,
school_subject_id_list multi,
school_material_type_id_list multi,
school_exam_id_list multi,
school_education_system_id_list multi,
school_umk_id_list multi,
school_exam_year_id_list multi,
school_purpose_id_list multi,
seo_title string attribute,
main_category_id_list multi,
category_id_list multi,
tbk_id_list multi,
ekn_id_list multi,
author_id_list multi,
translator_id_list multi,
illustrator_id_list multi,
publisher_series_id integer,
publisher_id integer,
publisher_brand_id integer,
manufacturer_brand_id integer,
literature_work_cycle_id integer,
literature_work_cycle_volume_id integer,
age_restriction bigint,
binding_id integer,
tag_id_list multi,
product_set_id_list multi,
product_type_id integer,
article_number_id integer,
stationery_brush_shape_id_list multi,
stationery_brush_material_id_list multi,
stationery_brush_number_id_list multi,
stationery_painting_technique_id_list multi,
stationery_lead_hardness_id_list multi,
stationery_format_id_list multi,
stationery_line_type_id_list multi,
stationery_ink_color_id_list multi,
stationery_lead_diameter_id_list multi,
stationery_case_shape_id_list multi,
stationery_mechanism_type_id_list multi,
stationery_diameter_id_list multi,
stationery_feature_id_list multi,
stationery_colors_quantity_id_list multi,
stationery_gender_id_list multi,
stationery_length_id_list multi,
stationery_staple_number_id_list multi,
stationery_stapler_number_id_list multi,
stationery_material_id_list multi,
stationery_punched_sheets_quantity_id_list multi,
stationery_pen_thickness_id_list multi,
stationery_mounting_type_id_list multi,
stationery_pen_tip_shape_id_list multi,
stationery_ink_base_id_list multi,
stationery_calculator_capacity_id_list multi,
stationery_calendar_year_id_list multi,
stationery_calendar_type_id_list multi,
stationery_calendar_subject_id_list multi,
stationery_clasp_type_id_list multi,
stationery_compartments_quantity_id_list multi,
stationery_cover_binding_id_list multi,
stationery_package_type_id_list multi,
stationery_cover_surface_id_list multi,
stationery_universal_id_list multi,
stationery_sheets_quantity_id_list multi,
stationery_color_id_list multi,
stationery_volume_id_list multi,
comic_universal_id_list multi,
comic_character_id_list multi,
comic_genre_id_list multi,
comic_series_id_list multi,
comic_type_id_list multi,
comic_line_id_list multi,
comic_section_id_list multi,
comic_subject_id_list multi,
game_player_quantity_id_list multi,
game_use_case_id_list multi,
game_skill_id_list multi,
game_audience_id_list multi,
game_child_age_id_list multi,
game_series_id_list multi,
game_duration_id_list multi,
constructor_detail_quantity_id_list multi,
constructor_nation_id_list multi,
constructor_equipment_type_id_list multi,
souvenir_reason_id_list multi,
souvenir_format_id_list multi,
souvenir_set_quantity_id_list multi,
toy_type_id_list multi,
toy_height_id_list multi,
gift_hobby_id_list multi,
gift_for_children_id_list multi,
gift_new_year_id_list multi,
gift_section_id_list multi,
gift_books_on_interest_id_list multi,
product_collection_id_list multi,
active_shops_id_list multi,
active_cities_id_list multi,
active_shop_brands_id_list multi,
all_shops_id_list multi,
all_cities_id_list multi,
all_shop_brands_id_list multi,
videos json,
attended_foreign_agents json,
synchronized_at timestamp
) min_prefix_len='3' index_exact_words='1' charset_table='0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+0401->U+0435, U+0451->U+0435, U+401->U+0435, U+451->U+0435' ignore_chars='U+002E, U+002D, U+005C' morphology='lemmatize_ru_all, lemmatize_en_all' min_stemming_len='4' expand_keywords='1'
SHOW TABLE products STATUS (снимал не в момент падения):
Variable_name |Value |
---------------------------+------------------------------------------------------------------------------------------------------------+
index_type |rt |
indexed_documents |2390275 |
indexed_bytes |213638098412 |
ram_bytes |2735527793 |
disk_bytes |5819804022 |
disk_mapped |2772596627 |
disk_mapped_cached |2726916096 |
disk_mapped_doclists |0 |
disk_mapped_cached_doclists|0 |
disk_mapped_hitlists |0 |
disk_mapped_cached_hitlists|0 |
killed_documents |787810 |
killed_rate |24.78% |
ram_chunk |8339465 |
ram_chunk_segments_count |3 |
disk_chunks |64 |
mem_limit |134217728 |
mem_limit_rate |95.00% |
ram_bytes_retired |0 |
locked |0 |
tid |0 |
tid_saved |0 |
query_time_1min |{"queries":18616, "avg_sec":0.004, "min_sec":0.000, "max_sec":0.909, "pct95_sec":0.013, "pct99_sec":0.033} |
query_time_5min |{"queries":90507, "avg_sec":0.005, "min_sec":0.000, "max_sec":5.269, "pct95_sec":0.011, "pct99_sec":0.047} |
query_time_15min |{"queries":262076, "avg_sec":0.005, "min_sec":0.000, "max_sec":5.668, "pct95_sec":0.011, "pct99_sec":0.099} |
query_time_total |{"queries":6471769, "avg_sec":0.003, "min_sec":0.000, "max_sec":5.668, "pct95_sec":0.009, "pct99_sec":0.022}|
found_rows_1min |{"queries":18616, "avg":3924, "min":0, "max":667767, "pct95":24413, "pct99":44533} |
found_rows_5min |{"queries":90507, "avg":3797, "min":0, "max":1918321, "pct95":26761, "pct99":60236} |
found_rows_15min |{"queries":262076, "avg":3915, "min":0, "max":1918321, "pct95":26764, "pct99":79963} |
found_rows_total |{"queries":6471769, "avg":3079, "min":0, "max":1918321, "pct95":670, "pct99":36173} |
SHOW VERSION
Component|Version |
---------+--------------------------------+
Daemon |6.3.6 593045790@24080214 |
Columnar |columnar 2.3.0 88a01c3@24052206 |
Secondary|secondary 2.3.0 88a01c3@24052206|
KNN |knn 2.3.0 88a01c3@24052206 |
Buddy |buddy v2.3.12 |
SHOW GLOBAL VARIABLES:
Variable_name |Value |
----------------------------+--------------+
autocommit |1 |
auto_optimize |1 |
optimize_cutoff |64 |
collation_connection |libc_ci |
query_log_format |sphinxql |
session_read_only |0 |
log_level |info |
max_allowed_packet |134217728 |
character_set_client |utf8 |
character_set_connection |utf8 |
grouping_in_utc |0 |
timezone |/etc/localtime|
last_insert_id | |
pseudo_sharding |1 |
secondary_indexes |1 |
accurate_aggregation |0 |
distinct_precision_threshold|3500 |
threads_ex_effective | |
thread_stack |16777216 |
threads_ex | |
SHOW STATUS
Counter |Value |
---------------------+------------------------------------------------------------------------------------------------------------------------------------------+
uptime |20689 |
connections |6493809 |
maxed_out |0 |
version |6.3.6 593045790@24080214 (columnar 2.3.0 88a01c3@24052206) (secondary 2.3.0 88a01c3@24052206) (knn 2.3.0 88a01c3@24052206) (buddy v2.3.12)|
mysql_version |6.3.6 593045790@24080214 (columnar 2.3.0 88a01c3@24052206) (secondary 2.3.0 88a01c3@24052206) (knn 2.3.0 88a01c3@24052206) |
command_search |28737827 |
command_excerpt |0 |
command_update |0 |
command_keywords |0 |
command_persist |0 |
command_status |11791 |
command_flushattrs |0 |
command_sphinxql |0 |
command_ping |0 |
command_delete |2191 |
command_set |16 |
command_insert |0 |
command_replace |2626 |
command_commit |0 |
command_suggest |211 |
command_json |0 |
command_callpq |0 |
command_cluster |0 |
command_getfield |0 |
agent_connect |0 |
agent_tfo |0 |
agent_retry |0 |
queries |28740003 |
dist_queries |0 |
workers_total |126 |
workers_active |112 |
workers_clients |32 |
workers_clients_vip |0 |
work_queue_length |220 |
load |56.13 64.49 65.66 |
load_primary |0.15 0.28 0.27 |
load_secondary |1.97 6.11 6.77 |
query_wall |147673.980 |
query_cpu |OFF |
dist_wall |0.000 |
dist_local |0.000 |
dist_wait |0.000 |
query_reads |OFF |
query_readkb |OFF |
query_readtime |OFF |
avg_query_wall |0.005 |
avg_query_cpu |OFF |
avg_dist_wall |0.000 |
avg_dist_local |0.000 |
avg_dist_wait |0.000 |
avg_query_reads |OFF |
avg_query_readkb |OFF |
avg_query_readtime |OFF |
qcache_max_bytes |16777216 |
qcache_thresh_msec |3000 |
qcache_ttl_sec |60 |
qcache_cached_queries|0 |
qcache_used_bytes |0 |
qcache_hits |0 |
Использование памяти в момент проблемы: 2024-12-30_16-13-46.png — Яндекс Диск
Использование свопа в момент проблемы: 2024-12-30_16-14-16.png — Яндекс Диск
Количество воркеров: 2024-12-30_16-16-40.png — Яндекс Диск
В searchd-логах ничего примечального не замечаем в момент падения