开源大数据集群部署（九）Ranger审计日志集成（solr）
1、下载solr安装包并解压包
tar -xzvf solr-8.11.2.gz
cd solr-8.11.2
执行安装脚本
./bin/install_solr_service.sh /opt/solr-8.11.2.tgz
安装后，会在/etc/default/ 下生成solr.in.sh文件。
2、在rangeradmin下生成solr相关配置
cd /opt/ranger-2.3.0-admin/contrib/solr_for_audit_setup/
编辑install.properties
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#Note:
#1. This file is sourced from setup.sh, so make sure there are no spaces after the "="
#2. For variable with file path, please provide full path
#!/bin/bash
#JAVA_HOME to be used by Solr. Solr only support JDK 1.7 and above. If JAVA_HOME is not set
#in the env, then please set it here
#JAVA_HOME=
#The operating system (linux) user used by Solr process. You need to run Solr as the below user and group
SOLR_USER=solr
SOLR_GROUP=solr
#How long to keep the audit logs. Please note, audit records grows very rapidly. Make sure to
#allocate enough memory and disk space to the server running Solr.
MAX_AUDIT_RETENTION_DAYS=90
#If you want this script to install Solr, set the value to true. If it is already installed, then set this to false
#If it is true, then it will download and install it.
#NOTE: If you want the script to install Solr, then this script needs to be executed as root.
SOLR_INSTALL=false
### BEGIN: if SOLR_INSTALL==true ###
#Location to download Solr. If SOLR_INSTALL is true, then SOLR_DOWNLOAD_URL is mandatory
#For open source version, pick a mirror from below. Recommended versions are Apache Solr 5.2.1 or above
#http://lucene.apache.org/solr/mirrors-solr-latest-redir.html
#Note: If possible, use the link from one of the mirror site
#SOLR_DOWNLOAD_URL=http://archive.apache.org/dist/lucene/solr/5.2.1/solr-5.2.1.tgz
SOLR_DOWNLOAD_URL=http://172.16.104.165:8999/warehouse/solr/solr-8.11.2.tgz
### END: if SOLR_INSTALL==true ###
#The folder where Solr is installed. If SOLR_INSTALL=false, then Solr need to be preinstalled, else the setup will
#install at the below location
#Note: If you are using RPM from LucidWorks in HDP, then Solr is by default installed in the following location:
#SOLR_INSTALL_FOLDER=/opt/lucidworks-hdpsearch/solr
SOLR_INSTALL_FOLDER=/opt/solr
#The location for the Solr configuration for Ranger. This script copies required configuration and
#startup scripts to the $SOLR_RANGER_HOME folder.
#NOTE: In SolrCloud mode, the data folders are under this folder. So make sure this is on seperate drive
# with enough disk space. Have 1TB free disk space on this volume. Also regularly monitor available disk space
# for this volume
#SOLR_RANGER_HOME=/opt/solr/ranger_audit_server
SOLR_RANGER_HOME=/opt/solr/ranger_audit_server
#Port for Solr instance to be used by Ranger.
SOLR_RANGER_PORT=8983
#Standalone or SolrCloud. Valid values are "standalone" or "solrcloud"
SOLR_DEPLOYMENT=solrcloud
#### BEGIN: if SOLR_DEPLOYMENT=standalone ##########################
#Location for the data files. Make sure it has enough disk space. Since audits records can grow dramatically,
#please have 1TB free disk space for the data folder. Also regularly monitor available disk space for this volume
SOLR_RANGER_DATA_FOLDER=/opt/solr/ranger_audit_server/data
#### END: if SOLR_DEPLOYMENT=standalone ##########################
#### BEGIN: if SOLR_DEPLOYMENT=solrcloud ##########################
#Comma seperated list of of zookeeper host and path. Give fully qualified domain name for the host
#SOLR_ZK=localhost:2181/ranger_audits
SOLR_ZK=hd1:2181,hd2:2181,hd3:2181/ranger_audits
#Base URL of the Solr. Used for creating collections
SOLR_HOST_URL=http://`hostname -f`:${SOLR_RANGER_PORT}
#Number of shards
SOLR_SHARDS=1
#Number of replication
SOLR_REPLICATION=1
#### END: if SOLR_DEPLOYMENT=solrcloud ##########################
#Location for the log file. Please note that "solr" or the process owner should have write permission
#to log folder
#SOLR_LOG_FOLDER=logs
SOLR_LOG_FOLDER=/var/log/solr/ranger_audits
SOLR_RANGER_COLLECTION=ranger_audits
#Memory for Solr. Both min and max memory to the java process are set to this value.
#Note: In production, please assign enough memory. It is recommended to have at least 2GB RAM.
# Higher the RAM, the better. Solr core can take upto 32GB. For dev test you can use 512m
#SOLR_MAX_MEM=2g
#SOLR_MAX_MEM=512m
SOLR_MAX_MEM=2g
配置说明
Ø /opt/solr/ranger_audit_server #生成配置文件的路径
Ø SOLR_INSTALL_FOLDER=/opt/solr #solr安装路径
Ø SOLR_DEPLOYMENT=SolrCloud #solr模式为分布式模式
Ø SOLR_ZK=hd1:2181,hd2:2181,hd3:2181/ranger_audits #zk地址及solrznode
执行生成脚本
./setup.sh cd /opt/solr/ranger_audit_server/scripts/ 可以查看到对应配置
将生成的solr.in.sh复制到/etc/default/下
cp solr.in.sh /etc/default/
在zk中生成solr的znode
bash add_ranger_audits_conf_to_zk.sh
启动solr
service solr start
默认日志路径
/var/log/solr/ranger_audits
查看端口是否启动成功
ss -tunlp | grep 8983
查看页面
http://hd1.dtstack.com:8983/solr/#/
创建audit 索引
cd /opt/solr/ranger_audit_server/scripts
修改create_ranger_audits_collection.sh脚本，添加端口8983
bash create_ranger_audits_collection.sh
成功后，可以在页面看到对应名称
修改rangeradmin的audit的配置
cd /opt/ranger-2.3.0-admin/
修改install.properties
audit_store=solr # * audit_solr_url URL to Solr. E.g. http://<solr_host>:6083/solr/ranger_audits audit_solr_urls=http://hd1.dtstack.com:8983/solr/ranger_audits audit_solr_user= audit_solr_password= audit_solr_zookeepers=hd1:2181,hd2:2181,hd3:2181/ranger_audits audit_solr_collection_name=ranger_audits #solr Properties for cloud mode audit_solr_config_name=ranger_audits audit_solr_configset_location= audit_solr_no_shards=1 audit_solr_no_replica=1 audit_solr_max_shards_per_node=1 audit_solr_acl_user_list_sasl=solr,infra-solr audit_solr_bootstrap_enabled=true
执行./setup.sh ,生成配置，重启rangeradmin
ranger-admin restart
如果一切正常，则在页面可以看到审计数据
