记录一次使用Telegrafa处理Influxdb v2磁盘高IO问题

{"level":"error","timestamp":"2025-10-08T00:12:55.659+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:13:10.292+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:13:25.992+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:13:42.502+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:14:00.116+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:14:15.766+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:14:27.656+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:14:39.615+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T00:14:50.310+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T16:25:26.512+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T16:31:58.866+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T16:32:09.139+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T16:35:59.374+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T16:36:09.923+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T16:36:20.206+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-08T16:36:30.885+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-09T00:10:16.687+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-09T00:10:28.356+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-09T00:10:44.493+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-09T00:10:58.603+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-09T00:11:12.604+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-09T00:11:29.668+0800","caller":"jtdata/jtdata.go:79","msg":"internal error: unexpected error writing points to database: timeout"}{"level":"error","timestamp":"2025-10-09T00:11:50.172+0800","caller":"jtdata/jtdata.go:79","msg":"Post \"http://localhost:8086/api/v2/write?bucket=jt&org=GTEX&precision=ns\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}

influxd inspect report-db --db-path /var/lib/influxdb/.influxdbv2/engine/data/d8390b10d2ccb963bucket             retention policy measurement series------             ---------------- ----------- ------"d8390b10d2ccb963""autogen"        "Data"      63488581"d8390b10d2ccb963""autogen"        "GNSS"      32893"d8390b10d2ccb963""autogen"        "STATE"     43830"d8390b10d2ccb963""autogen"                    63519728"d8390b10d2ccb963"                              63519728Total (est.)                                    63519728

ubuntu安装curl --silent --location -O https://repos.influxdata.com/influxdata-archive.keygpg --show-keys --with-fingerprint --with-colons ./influxdata-archive.key 2>&1 \| grep -q '^fpr:\+24C975CBA61A024EE1B631787C3D57159FC2F927:$' \&& cat influxdata-archive.key \| gpg --dearmor \| sudo tee /etc/apt/keyrings/influxdata-archive.gpg > /dev/null \&& echo 'deb [signed-by=/etc/apt/keyrings/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main' \| sudo tee /etc/apt/sources.list.d/influxdata.listsudo apt-get update && sudo apt-get install telegrafredhat&centos&rocky安装cat <<EOF | sudo tee /etc/yum.repos.d/influxdata.repo[influxdata]name = InfluxData Repository - Stablebaseurl = https://repos.influxdata.com/stable/$basearch/mainenabled = 1gpgcheck = 1gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-influxdataEOFsudo yum install telegraf编辑配置文件，设置缓冲批量提交双写cd /etc/telegrafvim double_wirte_test.conf# Configuration for telegraf agent[agent]  ## Default data collection interval for all inputs  interval = "10s"  ## Rounds collection interval to 'interval'  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.  round_interval = true  ## Telegraf will send metrics to outputs in batches of at most  ## metric_batch_size metrics.  ## This controls the size of writes that Telegraf sends to output plugins.  metric_batch_size = 1000  ## Maximum number of unwritten metrics per output.  Increasing this value  ## allows for longer periods of output downtime without dropping metrics at the  ## cost of higher maximum memory usage.  metric_buffer_limit = 10000  ## Collection jitter is used to jitter the collection by a random amount.  ## Each plugin will sleep for a random time within jitter before collecting.  ## This can be used to avoid many plugins querying things like sysfs at the  ## same time, which can have a measurable effect on the system.  collection_jitter = "0s"  ## Default flushing interval for all outputs. Maximum flush_interval will be  ## flush_interval + flush_jitter  flush_interval = "10s"  ## Jitter the flush interval by a random amount. This is primarily to avoid  ## large write spikes for users running a large number of telegraf instances.  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s  flush_jitter = "0s"  ## By defaultor when set to "0s", precision will be set to the same  ## timestamp order as the collection interval, with the maximum being 1s.  ##   ie, when interval = "10s", precision will be "1s"  ##       when interval = "250ms", precision will be "1ms"  ## Precision will NOT be used for service inputs. It is up to each individual  ## service input to set the timestamp at the appropriate precision.  ## Valid time units are "ns", "us" (or"µs"), "ms", "s".  precision = ""  ## Log at debug level.  #debug = true  ## Log only error level messages.  # quiet = false  ## Log target controls the destination for logs and can be one of "file",  ## "stderr"or, on Windows, "eventlog".  When set to "file", the output file  ## is determined by the "logfile" setting.  # logtarget = "file"  ## Name of the file to be logged to when using the "file" logtarget.  If set to  ## the empty string then logs are written to stderr.  # logfile = ""  ## The logfile will be rotated after the time interval specified.  When set  ## to 0 no time based rotation is performed.  Logs are rotated only when  ## written to, if there is no log activity rotation may be delayed.  # logfile_rotation_interval = "0d"  ## The logfile will be rotated when it becomes larger than the specified  ## size.  When set to 0 no size based rotation is performed.  # logfile_rotation_max_size = "0MB"  ## Maximum number of rotated archives to keep, any older logs are deleted.  ## If set to -1, no archives are removed.  # logfile_rotation_max_archives = 5  ## Pick a timezone to use when logging or type 'local'for local time.  ## Example: America/Chicago  # log_with_timezone = ""  ## Override default hostname, if empty use os.Hostname()  hostname = ""  ## If set to true, do no set the "host" tag in the telegraf agent.  omit_hostname = true[[inputs.influxdb_v2_listener]]  ## Address and port to host InfluxDB listener on  ## (Double check the port. Could be 9999ifusing OSS Beta)  service_address = ":8087"  ## Maximum allowed HTTP request body size in bytes.  ## 0 means to use the default of 32MiB.  # max_body_size = "32MiB"  ## Optional tag to determine the bucket.  ## If the write has a bucket in the query string then it will be kept in this tag name.  ## This tag can be used in downstream outputs.  ## The default value of nothing means it will be off and the database will not be recorded.  # bucket_tag = ""  ## Set one or more allowed client CA certificate file names to  ## enable mutually authenticated TLS connections  # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]  ## Add service certificate and key  # tls_cert = "/etc/telegraf/cert.pem"  # tls_key = "/etc/telegraf/key.pem"  ## Optional token to accept for HTTP authentication.  ## You probably want to make sure you have TLS configured above forthis.  #token = "bwRR7x-Xw2gZEKuU7U2Xs8siwEr6AQYOZXyrmgPOeAQZFnH2kGZ53hCrROpZq_xUIv8j_nomYgnDn4R3egkhMQ=="  token = "To23iYgAsdALSlYRseYZijj3LR7IQatDQRbL-VoQXVGiNwfMcWqc2GEcmbGFhQQ9Cl8n8nM-LKjpbr3bN0jWxw=="  ## Influx line protocol parser  ## 'internal' is the default. 'upstream' is a newer parser that is faster  ## and more memory efficient.  parser_type = "upstream"[[outputs.influxdb_v2]]  # 第二个 InfluxDB v2 实例  urls = ["http://主库ip地址:8086"] # 请替换为你的第三个实例地址  token = "pDbRtCHdCuleCSzgyQ_kxJc2KxGcOcnbkemaq1Cq3AHwueOhQ6QxbMbeJNH11vbg5DSY5CBjRgb54ab9Mn2vFg=="  organization = "GTEX"  bucket = "jt"  metric_batch_size = 10000  # 增大批次大小  metric_buffer_limit = 1000000 # 增大缓冲区  flush_interval = "15s" # 缩短刷新间隔  content_encoding = "gzip" #启用gzip压缩  concurrent_writes = 4 #启用多线程  influx_uint_support = true #启用支持unsigned类型字段  [outputs.influxdb_v2.tags]    influxdb_instance = "primary"[[outputs.influxdb_v2]]  # 第一个 InfluxDB v2 实例  urls = ["http://备库ip地址:28086"] # 请替换为你的第二个实例地址  token = "OlXD8YHa4ZJgk5KgGoqkBPaMKXIMB_SPppgSirgfNaB9ciU6kfot-NTk2NtDjkxxqBf1YNyKtNenv9HD12kCbQ=="  organization = "GTEX"  bucket = "jt"  metric_batch_size = 10000  # 增大批次大小  metric_buffer_limit = 1000000 # 增大缓冲区  flush_interval = "15s" # 缩短刷新间隔  content_encoding = "gzip" #启用gzip压缩  concurrent_writes = 4 #启用多线程  influx_uint_support = true #启用支持unsigned类型字段  [outputs.influxdb_v2.tags]    influxdb_instance = "secondary"启动telegrafcd /etc/telegrafnohup telegraf --config /etc/telegraf/double_wirte_test.conf &启动后telegraf会监听8087端口，程序客户端服务就可以直接连接http://influxdb服务器ip:8087，进行数据写入，telegraf收到数据后缓冲到缓冲区，将客户端写入的line格式数据按批提交到声明outputs.influxdb_v2的influxdb数据库中。