ElasticSearch安装

# ElasticSearch
## 说明
- 版本 7.11.2
- 已配置ISCSI存储
- 默认使用集群模式,为扩容准备
- 配置文档 https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html

## 环境信息
- 127.0.0.1
- ISCSI 存储4T 挂载至 /data `fstab需要加   _netdev`

## 安装

```shell
# 安装包
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.11.2-x86_64.rpm
rpm -ivh elasticsearch-7.11.2-x86_64.rpm
# 默认的安装目录 /usr/share/elasticsearch
echo 'export PATH=$PATH:/usr/share/elasticsearch/bin' >> /etc/profile
source /etc/profile
```

```shell
# 创建数据目录
mkdir -p /data/elasticsearch/{data,tmp,logs,snapshorts}
chown -R elasticsearch:elasticsearch /data/elasticsearch
```

```shell
# 修改配置文件
cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.old
cp /etc/elasticsearch/jvm.options /etc/elasticsearch/jvm.options.old

# 修改Elasticsearch配置
cat > /etc/elasticsearch/elasticsearch.yml <<EOF
cluster.name: ES-cluster
node.name: ES-master
node.master: true
# 从节点应为
# node.master: false
path.data: /data/elasticsearch/data
path.repo: /data/elasticsearch/snapshorts
path.logs: /data/elasticsearch/logs
network.host: 127.0.0.1
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*" 
discovery.seed_hosts: ["127.0.0.1"]
cluster.initial_master_nodes: ["ES-master"]

EOF
```

```shell
# 修改jvm虚拟机配置
cat > /etc/elasticsearch/jvm.options <<EOF
-Xms6g
-Xmx6g
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=/data/elasticsearch/tmp
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/data/elasticsearch/data
-XX:ErrorFile=/data/elasticsearch/logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:/data/elasticsearch/logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=/data/elasticsearch/logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m

EOF

cat >> /etc/sysconfig/elasticsearch <<EOF
ES_JAVA_OPTS="-Djna.tmpdir=/data/elasticsearch/tmp"
EOF
```

```shell
# 启动服务
systemctl enable elasticsearch
systemctl start elasticsearch

```

## 安全配置
- 默认的配置是没有任何安全防护的，如果不配置防火墙的话任何人都可以访问你的数据，所以必须配置认证功能！

```shell
mkdir /etc/elasticsearch/sslcerts
# 使用自带工具生成证书
elasticsearch-certutil cert --ip ${es安装IP，可以用逗号分隔多个节点的IP} -out /etc/elasticsearch/sslcerts/elastic-certificates.p12 -pass ""

# openssl生成客户端连接es的证书
openssl pkcs12 -in elastic-certificates.p12 -out elasticsearch.pem -clcerts -nokeys

# 生成客户端连接es的证书（可选，可以直接用上面的证书）
/usr/share/elasticsearch/bin/elasticsearch-certutil cert --pem /etc/elasticsearch/sslcerts/elastic-certificates.p12

# 拷贝证书
cp /usr/share/elasticsearch/certificate-bundle.zip /root/

# 由于是自签名证书，所以需要将证书文件放入linux的ca bundle里面
cat /etc/kibana/sslcerts/elasticsearch.pem |sed -n "/----BEGIN/,/----END/p" >> /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem

# 配置elastcisearch使用ssl加密传输数据
cat >> /etc/elasticsearch/elasticsearch.yml <<EOF
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /etc/elasticsearch/sslcerts/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /etc/elasticsearch/sslcerts/elastic-certificates.p12
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: /etc/elasticsearch/sslcerts/elastic-certificates.p12
xpack.security.http.ssl.truststore.path: /etc/elasticsearch/sslcerts/elastic-certificates.p12
xpack.security.authc.api_key.enabled: true

EOF

chown -R elasticsearch:elasticsearch /etc/elasticsearch/
```

```shell
# 生成elasticsearch 用户及密码
elasticsearch-setup-passwords auto # 可以使用interactive来手动输入每个用户的密码

# 记住下面生成的账号密码，后续会使用到。如果忘了也可以通过命令行更改密码。
# ---------------------------------------
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,kibana_system,logstash_system,beats_system,remote_monitoring_user.
The passwords will be randomly generated and printed to the console.
Please confirm that you would like to continue [y/N]y

Changed password for user apm_system
PASSWORD apm_system = IamAFakePasswordLOL

Changed password for user kibana_system
PASSWORD kibana_system = IamAFakePasswordLOL

Changed password for user kibana
PASSWORD kibana = IamAFakePasswordLOL

Changed password for user logstash_system
PASSWORD logstash_system = IamAFakePasswordLOL

Changed password for user beats_system
PASSWORD beats_system = IamAFakePasswordLOL

Changed password for user remote_monitoring_user
PASSWORD remote_monitoring_user = IamAFakePasswordLOL

Changed password for user elastic
PASSWORD elastic = IamAFakePasswordLOL
# ---------------------------------------

```

# Kibana
## 说明
- 版本 7.11.2
- 配置文档 https://www.elastic.co/guide/en/kibana/current/settings.html

## 安装

```shell
# 安装包
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.11.2-x86_64.rpm
rpm -ivh  kibana-7.11.2-x86_64.rpm

cp /etc/kibana/kibana.yml /etc/kibana/kibana.yml.old

cat > /etc/kibana/kibana.yml <<EOF
server.port: 5601
server.host: "127.0.0.1"
elasticsearch.hosts: ["http://127.0.0.1:9200"]
kibana.index: ".kibana"
elasticsearch.requestTimeout: 60000
i18n.locale: "zh-CN"
elasticsearch.username: "kibana_system"
# 下面的密码使用上一步“安全配置”中生成的密码
elasticsearch.password: "IamAFakePasswordLOL"
# 安全配置
elasticsearch.ssl.certificateAuthorities: ["/etc/kibana/sslcerts/elasticsearch.pem"]
elasticsearch.ssl.verificationMode: none
EOF

systemctl enable kibana
systemctl start kibana

```

# Curator - 索引生命周期管理工具
## 说明
- 版本 5.8
- python 3.6
- 文档链接 https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html

## 安装

```shell
pip3 install -i https://mirrors.sustech.edu.cn/pypi/simple elasticsearch-curator
# 如果PATH环境变量不包含/usr/local/bin
echo "export PATH=\$PATH:/usr/local/bin" >> /erc/profile
source /etc/profile
# 检查是否安装成功
curator_cli --help
curator --help

# curator 配置
cat >> ~/.curator/curator.yml <<EOF
---
# Remember, leave a key empty if there is no value.  None will be a string,
# not a Python "NoneType"
# 即使没有配置下面的变量也要将变量名输入进去，这样变量名会保存为字符串类型而不是None类型
# 记得输入上面我们“elasticsearch-安全配置”中生成的密码

client:
  hosts:
    - 127.0.0.1
  port: 9200
  url_prefix:
  use_ssl: False
  certificate:
  client_cert:
  client_key:
  ssl_no_validate: False
  username: elastic
  password: 
  timeout: 30
  master_only: False

logging:
  loglevel: INFO
  logfile:
  logformat: default
  # 无特殊需求配置为空
  blacklist: []
EOF

```

## 使用curator客户端
- 相关链接
    - [Actions](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/actions.html)
    - [Filters](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filters.html)
	
```shell
# 使用客户端需要配置动作文件（actionfile）
# 下面是一个冻结时间为一个月以上的旧索引的动作文件。
cat > freeze-index-older-than-one-month.yml <<EOF
actions:
  freeze-dxggyw:
    action: freeze
    description: >-
      freeze indices older than 1 month.
    options:
      disable_action: False
    filters:
    - filtertype: pattern
      kind: prefix
      value: dxggyw-
      exclude:
    - filtertype: age
      source: name
      direction: older
      timestring: '%Y.%m%'
      unit: months
      unit_count: 1
EOF

# 为了做实验改为了清理一天前的索引，测试可行性。去掉--dry-run以实际执行
[root@ELK01 ~]# curator --dry-run freeze-index-older-than-one-month.yml 
2021-03-21 00:29:15,843 INFO      Preparing Action ID: freeze-dxggyw, "freeze"
2021-03-21 00:29:15,843 INFO      Creating client object and testing connection
2021-03-21 00:29:15,845 INFO      Instantiating client object
2021-03-21 00:29:15,845 INFO      Testing client connectivity
2021-03-21 00:29:15,850 INFO      GET http://127.0.0.1:9200/ [status:200 request:0.004s]
2021-03-21 00:29:15,850 INFO      Successfully created Elasticsearch client object with provided settings
2021-03-21 00:29:15,851 INFO      GET http://127.0.0.1:9200/ [status:200 request:0.002s]
2021-03-21 00:29:15,852 INFO      Trying Action ID: freeze-dxggyw, "freeze": Freeze indices older than 30 days but younger than 60 days (based on index name), for logstash- prefixed indices.
2021-03-21 00:29:15,854 INFO      GET http://127.0.0.1:9200/_all/_settings?expand_wildcards=open%2Cclosed [status:200 request:0.003s]
2021-03-21 00:29:15,856 INFO      GET http://127.0.0.1:9200/ [status:200 request:0.001s]
2021-03-21 00:29:15,862 INFO      GET http://127.0.0.1:9200/_cluster/state/metadata/.apm-agent-configuration,.apm-custom-link,.kibana-event-log-7.11.2-000001,.kibana_1,.kibana_task_manager_1,dxggywyjs01-2021.03 [status:200 request:0.006s]
2021-03-21 00:29:15,866 INFO      GET http://127.0.0.1:9200/.apm-agent-configuration,.apm-custom-link,.kibana-event-log-7.11.2-000001,.kibana_1,.kibana_task_manager_1,dxggywyjs01-2021.03/_stats/store,docs [status:200 request:0.002s]
2021-03-21 00:29:15,869 INFO      DRY-RUN MODE.  No changes will be made.
2021-03-21 00:29:15,869 INFO      (CLOSED) indices may be shown that may not be acted on by action "freeze".
2021-03-21 00:29:15,869 INFO      Action ID: freeze-dxggyw, "freeze" completed.
2021-03-21 00:29:15,869 INFO      Job completed.

```

## 使用curator_cli工具

```shell
# 查看index
curator_cli --host 127.0.0.1  --port 9200 show_indices

# 用起来好麻烦，迟点研究
# 复杂工作建议使用actionfiles
```

# Logstash

# Beats
## Filebeat
### 安装配置
- 版本 7.2.0`旧日志服务器沿用`
- 文档 https://www.elastic.co/guide/en/beats/filebeat/current/index.html

```shell
# 7.2.0
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.2.0-x86_64.rpm
rpm -ivh filebeat-7.2.0-x86_64.rpm 
# 7.11.2
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.11.2-x86_64.rpm
rpm -ivh filebeat-7.11.2-x86_64.rpm

cp /etc/filebeat/filebeat.yml /etc/filebeat/filebeat.yml.old

cat > /etc/filebeat/filebeat.yml <<EOF
filebeat.inputs:
- type: log
  enabled: false
  paths:
    - /var/log/*.log
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
setup.template.settings:
  index.number_of_shards: 1
name: register-nginx01
setup.kibana:
  host: "1.1.1.1:5601"
output.elasticsearch:
  hosts: ["1.1.1.1:9200"]
  pipeline: geoip-info
  protocol: "https"
  username: "elastic"
  password: "IamAFakePasswordLOL"
ssl.certificate_authorities: ["/etc/filebeat/sslcerts/elasticsearch.pem"]
ssl.verification_mode: none
setup.ilm.enabled: auto
setup.ilm.rollover_alias: "${indexname}"
setup.ilm.pattern: "{now/d}-000001"
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
EOF

filebeat modules enable nginx
filebeat setup
systemctl start filebeat
systemctl enable filebeat
```

## Metricbeat
### 安装配置
- 版本 7.11.2

```shell
curl -L -O https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-7.11.2-x86_64.rpm
sudo rpm -vi metricbeat-7.11.2-x86_64.rpm

cp /etc/metricbeat/metricbeat.yml /etc/metricbeat/metricbeat.yml.old

```