elasticsearch+logstash并使用java代码实现日志检索

2023-07-05 07:42:03 312

为了项目日志不被泄露，数据展示不采用Kibana

1、环境准备

1.1创建普通用户

#创建用户
useraddquerylog

#设置密码
passwdqueylog

#授权sudo权限
查找sudoers文件位置
whereissudoers
#修改文件为可编辑
chmod-vu+w/etc/sudoers
#编辑文件
vi/etc/sudoers
#收回权限
chmod-vu-w/etc/sudoers
#第一次使用sudo会有提示

WetrustyouhavereceivedtheusuallecturefromthelocalSystem
Administrator.Itusuallyboilsdowntothesethreethings:

#1)Respecttheprivacyofothers.
#2)Thinkbeforeyoutype.
#3)Withgreatpowercomesgreatresponsibility.

用户创建完成。

1.2安装jdk

suqueylog
cd/home/queylog
#解压jdk-8u191-linux-x64.tar.gz
tar-zxvfjdk-8u191-linux-x64.tar.gz
sudomvjdk1.8.0_191/opt/jdk1.8
#编辑/ect/profile
vi/ect/profile
exportJAVA_HOME=/opt/jdk1.8
exportJRE_HOME=$JAVA_HOME/jre
exportCLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
exportPATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
#刷新配置文件
source/ect/profile
#查看jdk版本
java-verion

1.3防火墙设置

#放行指定IP
firewall-cmd--permanent--add-rich-rule="rulefamily="ipv4"sourceaddress="172.16.110.55"accept"
#重新载入
firewall-cmd--reload

2、安装elasticsearch

2.1elasticsearch配置

注意：elasticsearch要使用普通用户启动要不然会报错

suqueylog
cd/home/queylog
#解压elasticsearch-6.5.4.tar.gz
tar-zxvfelasticsearch-6.5.4.tar.gz
sudomvelasticsearch-6.5.4/opt/elasticsearch
#编辑es配置文件
vi/opt/elasticsearch/config/elasticsearch.yml
#配置es的集群名称
cluster.name:elastic
#修改服务地址
network.host:192.168.8.224
#修改服务端口
http.port:9200

#切换root用户
suroot
#修改/etc/security/limits.conf追加以下内容
vi/etc/security/limits.conf
*hardnofile655360
*softnofile131072
*hardnproc4096
*softnproc2048

#编辑/etc/sysctl.conf，追加以下内容：
vi/etc/sysctl.conf
vm.max_map_count=655360
fs.file-max=655360

#保存后，重新加载：
sysctl-p

#切换回普通用户
suqueylog
#启动elasticsearch
./opt/elasticsearch/bin/elasticsearch
#测试
curlhttp://192.168.8.224:9200
#控制台会打印
{
"name":"L_dA6oi",
"cluster_name":"elasticsearch",
"cluster_uuid":"eS7yP6fVTvC8KMhLutOz6w",
"version":{
"number":"6.5.4",
"build_flavor":"default",
"build_type":"tar",
"build_hash":"d2ef93d",
"build_date":"2018-12-17T21:17:40.758843Z",
"build_snapshot":false,
"lucene_version":"7.5.0",
"minimum_wire_compatibility_version":"5.6.0",
"minimum_index_compatibility_version":"5.0.0"
},
"tagline":"YouKnow,forSearch"
}

2.2把elasticsearch作为服务进行管理

#切换root用户
suroot
#编写服务配置文件
vi/usr/lib/systemd/system/elasticsearch.service
[unit]
Description=Elasticsearch
Documentation=http://www.elastic.co
Wants=network-online.target
After=network-online.target

[Service]
Environment=ES_HOME=/opt/elasticsearch
Environment=ES_PATH_CONF=/opt/elasticsearch/config
Environment=PID_DIR=/opt/elasticsearch/config
EnvironmentFile=/etc/sysconfig/elasticsearch
WorkingDirectory=/opt/elasticsearch
User=queylog
Group=queylog
ExecStart=/opt/elasticsearch/bin/elasticsearch-p${PID_DIR}/elasticsearch.pid

#StandardOutputisconfiguredtoredirecttojournalctlsince
#someerrormessagesmaybeloggedinstandardoutputbefore
#elasticsearchloggingsystemisinitialized.Elasticsearch
#storesitslogsin/var/log/elasticsearchanddoesnotuse
#journalctlbydefault.Ifyoualsowanttoenablejournalctl
#logging,youcansimplyremovethe"quiet"optionfromExecStart.
StandardOutput=journal
StandardError=inherit

#Specifiesthemaximumfiledescriptornumberthatcanbeopenedbythisprocess
LimitNOFILE=65536

#Specifiesthemaximumnumberofprocess
LimitNPROC=4096

#Specifiesthemaximumsizeofvirtualmemory
LimitAS=infinity

#Specifiesthemaximumfilesize
LimitFSIZE=infinity

#Disabletimeoutlogicandwaituntilprocessisstopped
TimeoutStopSec=0

#SIGTERMsignalisusedtostoptheJavaprocess
KillSignal=SIGTERM

#SendthesignalonlytotheJVMratherthanitscontrolgroup
KillMode=process

#Javaprocessisneverkilled
SendSIGKILL=no

#WhenaJVMreceivesaSIGTERMsignalitexitswithcode143
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target


vi/etc/sysconfig/elasticsearch

elasticsearch#
#######################

#Elasticsearchhomedirectory
ES_HOME=/opt/elasticsearch

#ElasticsearchJavapath
JAVA_HOME=/home/liyijie/jdk1.8
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JAVA_HOMR/jre/lib

#Elasticsearchconfigurationdirectory
ES_PATH_CONF=/opt/elasticsearch/config

#ElasticsearchPIDdirectory
PID_DIR=/opt/elasticsearch/config

#############################
#ElasticsearchService#
#############################

#SysVinit.d
#Thenumberofsecondstowaitbeforecheckingifelasticsearchstartedsuccessfullyasadaemonprocess
ES_STARTUP_SLEEP_TIME=5

################################
#ElasticsearchProperties#
################################
#Specifiesthemaximumfiledescriptornumberthatcanbeopenedbythisprocess
#WhenusingSystemd,thissettingisignoredandtheLimitNOFILEdefinedin
#/usr/lib/systemd/system/elasticsearch.servicetakesprecedence
#MAX_OPEN_FILES=65536

#ThemaximumnumberofbytesofmemorythatmaybelockedintoRAM
#Setto"unlimited"ifyouusethe'bootstrap.memory_lock:true'option
#inelasticsearch.yml.
#WhenusingSystemd,LimitMEMLOCKmustbesetinaunitfilesuchas
#/etc/systemd/system/elasticsearch.service.d/override.conf.
#MAX_LOCKED_MEMORY=unlimited

#MaximumnumberofVMA(VirtualMemoryAreas)aprocesscanown
#WhenusingSystemd,thissettingisignoredandthe'vm.max_map_count'
#propertyissetatboottimein/usr/lib/sysctl.d/elasticsearch.conf
#MAX_MAP_COUNT=262144

#重新加载服务
systemctldaemon-reload
#切换普通用户
suqueylog
#启动elasticsearch
sudosystemctlstartelasticsearch
#设置开机自启动
sudosystemctlenableelasticsearch

3、安装logstash

3.1、logstash配置

suqueylog
cd/home/queylog
#解压logstash-6.5.4.tar.gz
tar-zxvflogstash-6.5.4.tar.gz
sudomvlogstash-6.5.4/opt/logstash
#编辑es配置文件
vi/opt/logstash/config/logstash.yml
xpack.monitoring.enabled:true
xpack.monitoring.elasticsearch.username:elastic
xpack.monitoring.elasticsearch.password:changeme
xpack.monitoring.elasticsearch.url:["http://192.168.8.224:9200"]
#在bin目录下创建logstash.conf
vi/opt/logstash/bin/logstash.conf
input{
#以文件作为来源
file{
#日志文件路径
path=>"/opt/tomcat/logs/catalina.out"
start_position=>"beginning"#(end,beginning)
type=>"isp"
}
}
#filter{
#定义数据的格式，正则解析日志（根据实际需要对日志日志过滤、收集）
#grok{
#match=>{"message"=>"%{IPV4:clientIP}|%{GREEDYDATA:request}|%{NUMBER:duration}"}
#}
#根据需要对数据的类型转换
#mutate{convert=>{"duration"=>"integer"}}
#}
#定义输出
output{
elasticsearch{
hosts=>"192.168.43.211:9200"#Elasticsearch默认端口
index=>"ind"
document_type=>"isp"
}
}
#给该用户授权
chownqueylog:queylog/opt/logstash
#启动logstash
./opt/logstash/bin/logstash-flogstash.conf

#安装并配置启动logstash后查看es索引是否创建完成
curlhttp://192.168.8.224:9200/_cat/indices

4、java代码部分

之前在SpringBoot整合ElasticSearch与Redis的异常解决

查阅资料，这个归纳的原因比较合理。
原因分析：程序的其他地方使用了Netty，这里指redis。这影响在实例化传输客户端之前初始化处理器的数量。实例化传输客户端时，我们尝试初始化处理器的数量。由于在其他地方使用Netty，因此已经初始化并且Netty会对此进行防范，因此首次实例化会因看到的非法状态异常而失败。

解决方案

在SpringBoot启动类中加入：

System.setProperty("es.set.netty.runtime.available.processors","false");

4.1、引入pom依赖


org.springframework.boot
spring-boot-starter-data-elasticsearch

4.2、修改配置文件

spring.data.elasticsearch.cluster-name=elastic
#restapi使用9200
#java程序使用9300
spring.data.elasticsearch.cluster-nodes=192.168.43.211:9300

4.3、对应的接口以及实现类

importorg.springframework.data.elasticsearch.annotations.Document;
importorg.springframework.data.elasticsearch.annotations.Field;

@Document(indexName="ind",type="isp")
publicclassBean{


@Field
privateStringmessage;

publicStringgetMessage(){
returnmessage;
}

publicvoidsetMessage(Stringmessage){
this.message=message;
}

@Override
publicStringtoString(){
return"Tomcat{"+
",message='"+message+'\''+
'}';
}
}

importjava.util.Map;
publicinterfaceIElasticSearchService{

Mapsearch(Stringkeywords,IntegercurrentPage,IntegerpageSize)throwsException;

	//特殊字符转义
defaultStringescape(Strings){
StringBuildersb=newStringBuilder();

for(inti=0;i
importorg.elasticsearch.index.query.BoolQueryBuilder;
importorg.elasticsearch.index.query.QueryBuilders;
importorg.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
importorg.slf4j.Logger;
importorg.slf4j.LoggerFactory;
importorg.springframework.beans.factory.annotation.Autowired;
importorg.springframework.data.domain.PageRequest;
importorg.springframework.data.elasticsearch.core.ElasticsearchTemplate;
importorg.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
importorg.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
importorg.springframework.stereotype.Service;

importjavax.annotation.Resource;
importjava.util.ArrayList;
importjava.util.HashMap;
importjava.util.List;
importjava.util.Map;

/**
*ElasticSearch实现类
*/
@Service
publicclassElasticSearchServiceImplimplementsIElasticSearchService{

Loggerlog=LoggerFactory.getLogger(ElasticSearchServiceImpl.class);

@Autowired
ElasticsearchTemplateelasticsearchTemplate;

@Resource
HighlightResultHelperhighlightResultHelper;

@Override
publicMapsearch(Stringkeywords,IntegercurrentPage,IntegerpageSize){
keywords=escape(keywords);
currentPage=Math.max(currentPage-1,0);

ListhighlightFields=newArrayList<>();
//设置高亮把查询到的关键字进行高亮
HighlightBuilder.Fieldmessage=newHighlightBuilder.Field("message").fragmentOffset(80000).numOfFragments(0).requireFieldMatch(false).preTags("").postTags("");
highlightFields.add(message);

HighlightBuilder.Field[]highlightFieldsAry=highlightFields.toArray(newHighlightBuilder
.Field[highlightFields.size()]);

//创建查询构造器
NativeSearchQueryBuilderqueryBuilder=newNativeSearchQueryBuilder();
//过滤按字段权重进行搜索查询内容不为空按关键字、摘要、其他属性权重
BoolQueryBuilderboolQueryBuilder=QueryBuilders.boolQuery();
queryBuilder.withPageable(PageRequest.of(currentPage,pageSize));
if(!MyStringUtils.isEmpty(keywords)){
boolQueryBuilder.must(QueryBuilders.queryStringQuery(keywords).field("message"));
}

queryBuilder.withQuery(boolQueryBuilder);

queryBuilder.withHighlightFields(highlightFieldsAry);

log.info("查询语句：{}",queryBuilder.build().getQuery().toString());

//查询
AggregatedPageresult=elasticsearchTemplate.queryForPage(queryBuilder.build(),Bean
.class,highlightResultHelper);
//解析结果
longtotal=result.getTotalElements();
inttotalPage=result.getTotalPages();
ListblogList=result.getContent();
Mapmap=newHashMap<>();
map.put("total",total);
map.put("totalPage",totalPage);
map.put("pageSize",pageSize);
map.put("currentPage",currentPage+1);
map.put("blogList",blogList);
returnmap;
}
importcom.alibaba.fastjson.JSONObject;
importorg.apache.commons.beanutils.PropertyUtils;
importorg.elasticsearch.action.search.SearchResponse;
importorg.elasticsearch.common.text.Text;
importorg.elasticsearch.search.SearchHit;
importorg.elasticsearch.search.fetch.subphase.highlight.HighlightField;
importorg.slf4j.Logger;
importorg.slf4j.LoggerFactory;
importorg.springframework.data.domain.Pageable;
importorg.springframework.data.elasticsearch.core.SearchResultMapper;
importorg.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
importorg.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
importorg.springframework.stereotype.Component;
importorg.springframework.util.StringUtils;

importjava.lang.reflect.InvocationTargetException;
importjava.util.ArrayList;
importjava.util.List;

/**
*ElasticSearch高亮配置
*/
@Component
publicclassHighlightResultHelperimplementsSearchResultMapper{

Loggerlog=LoggerFactory.getLogger(HighlightResultHelper.class);


@Override
publicAggregatedPagemapResults(SearchResponseresponse,Classclazz,Pageablepageable){
Listresults=newArrayList<>();
for(SearchHithit:response.getHits()){
if(hit!=null){
Tresult=null;
if(StringUtils.hasText(hit.getSourceAsString())){
result=JSONObject.parseObject(hit.getSourceAsString(),clazz);
}
//高亮查询
for(HighlightFieldfield:hit.getHighlightFields().values()){
try{
PropertyUtils.setProperty(result,field.getName(),concat(field.fragments()));
}catch(IllegalAccessException|InvocationTargetException|NoSuchMethodExceptione){
log.error("设置高亮字段异常：{}",e.getMessage(),e);
}
}
results.add(result);
}
}
returnnewAggregatedPageImpl(results,pageable,response.getHits().getTotalHits(),response
.getAggregations(),response.getScrollId());
}


publicTmapSearchHit(SearchHitsearchHit,Classclazz){
Listresults=newArrayList<>();
for(HighlightFieldfield:searchHit.getHighlightFields().values()){
Tresult=null;
if(StringUtils.hasText(searchHit.getSourceAsString())){
result=JSONObject.parseObject(searchHit.getSourceAsString(),clazz);
}
try{
PropertyUtils.setProperty(result,field.getName(),concat(field.fragments()));
}catch(IllegalAccessException|InvocationTargetException|NoSuchMethodExceptione){
log.error("设置高亮字段异常：{}",e.getMessage(),e);
}
results.add(result);
}
returnnull;
}

privateStringconcat(Text[]texts){
StringBuffersb=newStringBuffer();
for(Texttext:texts){
sb.append(text.toString());
}
returnsb.toString();
}
}

importorg.junit.Test;
importorg.junit.runner.RunWith;
importorg.slf4j.Logger;
importorg.slf4j.LoggerFactory;
importorg.springframework.beans.factory.annotation.Autowired;
importorg.springframework.boot.test.context.SpringBootTest;
importorg.springframework.test.context.junit4.SpringRunner;

@RunWith(SpringRunner.class)
@SpringBootTest(classes=CbeiIspApplication.class)
publicclassElasticSearchServiceTest{w

privatestaticLoggerlogger=LoggerFactory.getLogger(EncodePhoneAndCardTest.class);

@Autowired
privateIElasticSearchServiceelasticSearchService;

	@Test
publicResponseVOgetLog(){
try{
Mapsearch=elasticSearchService.search("Exception",1,10);
logger.info(JSON.toJSONString(search));
}catch(Exceptione){
e.printStackTrace();
}
}
例如：以上就是今天要讲的内容，本文仅仅简单介绍了elasticsearch跟logstash的使用，文章若有不当之处，欢迎评论指出~
到此这篇关于elasticsearch+logstash并使用java代码实现日志检索的文章就介绍到这了,更多相关elasticsearchlogstash日志检索内容请搜索毛票票以前的文章或继续浏览下面的相关文章希望大家以后多多支持毛票票！
声明：本文内容来源于网络，版权归原作者所有，内容由互联网用户自发贡献自行上传，本网站不拥有所有权，未作人工编辑处理，也不承担相关法律责任。如果您发现有涉嫌版权的内容，欢迎发送邮件至：czq8825#qq.com（发邮件时，请将#更换为@）进行举报，并提供相关证据，一经查实，本站将立刻删除涉嫌侵权内容。