基于python3抓取pinpoint应用信息入库
这篇文章主要介绍了基于python3抓取pinpoint应用信息入库,文中通过示例代码介绍的非常详细，对大家的学习或者工作具有一定的参考学习价值,需要的朋友可以参考下
Pinpoint是用Java编写的大型分布式系统的APM（应用程序性能管理）工具。受Dapper的启发，Pinpoint提供了一种解决方案，通过在分布式应用程序中跟踪事务来帮助分析系统的整体结构以及它们中的组件之间的相互关系.
pinpointapi:
- /applications.pinpoint获取applications基本信息
- /getAgentList.pinpoint获取对应applicationagent信息
- /getServerMapData.pinpoint获取对应app基本数据流信息
db.py
importmysql.connector classMyDB(object): """docstringforMyDB""" def__init__(self,host,user,passwd,db): self.host=host self.user=user self.passwd=passwd self.db=db self.connect=None self.cursor=None defdb_connect(self): """数据库连接 """ self.connect=mysql.connector.connect(host=self.host,user=self.user,passwd=self.passwd,database=self.db) returnself defdb_cursor(self): ifself.connectisNone: self.connect=self.db_connect() ifnotself.connect.is_connected(): self.connect=self.db_connect() self.cursor=self.connect.cursor() returnself defget_rows(self,sql): """查询数据库结果 :paramsql:SQL语句 :paramcursor:数据库游标 """ self.cursor.execute(sql) returnself.cursor.fetchall() defdb_execute(self,sql): self.cursor.execute(sql) self.connect.commit() defdb_close(self): """关闭数据库连接和游标 :paramconnect:数据库连接实例 :paramcursor:数据库游标 """ ifself.connect: self.connect.close() ifself.cursor: self.cursor.close()
pinpoint.py:
#-*-coding:utf-8-*-
'''
Copyright(c)2018,mersap
Allrightsreserved.
摘要:pinpoint.py
创建者:mersap
创建日期:2019-01-17
'''
importsys
importrequests
importtime
importdatetime
importjson
sys.path.append('../Golf')
importdb#db.py
PPURL="https://pinpoint.*******.com"
From_Time=datetime.datetime.now()+datetime.timedelta(seconds=-60)
To_Time=datetime.datetime.now()
From_TimeStamp=int(time.mktime(From_Time.timetuple()))*1000
To_TimeStamp=int(time.mktime(datetime.datetime.now().timetuple()))*1000
classPinPoint(object):
"""docstringforPinPoint"""
def__init__(self,db):
self.db=db
super(PinPoint,self).__init__()
"""获取pinpoint中应用"""
defget_applications(self):
'''returnapplicationdict
'''
applicationListUrl=PPURL+"/applications.pinpoint"
res=requests.get(applicationListUrl)
ifres.status_code!=200:
print("请求异常,请检查")
return
applicationLists=[]
forappinres.json():
applicationLists.append(app)
applicationListDict={}
applicationListDict["applicationList"]=applicationLists
returnapplicationListDict
defgetAgentList(self,appname):
AgentListUrl=PPURL+"/getAgentList.pinpoint"
param={
'application':appname
}
res=requests.get(AgentListUrl,params=param)
ifres.status_code!=200:
print("请求异常,请检查")
return
returnlen(res.json().keys()),json.dumps(list(res.json().keys()))
defupdate_servermap(self,appname,from_time=From_TimeStamp,
to_time=To_TimeStamp,serviceType='SPRING_BOOT'):
'''更新app上下游关系
:paramappname:应用名称
:paramserviceType:应用类型
:paramfrom_time:起始时间
:paramto_time:终止时间
:
'''
#https://pinpoint.*****.com/getServerMapData.pinpoint?applicationName=test-app&from=1547721493000&to=1547721553000&callerRange=1&calleeRange=1&serviceTypeName=TOMCAT&_=1547720614229
param={
'applicationName':appname,
'from':from_time,
'to':to_time,
'callerRange':1,
'calleeRange':1,
'serviceTypeName':serviceType
}
#serverMapUrl=PPURL+"/getServerMapData.pinpoint"
serverMapUrl="{}{}".format(PPURL,"/getServerMapData.pinpoint")
res=requests.get(serverMapUrl,params=param)
ifres.status_code!=200:
print("请求异常,请检查")
return
update_time=time.strftime('%Y-%m-%d%H:%M:%S',time.localtime(time.time()))
links=res.json()["applicationMapData"]["linkDataArray"]
forlinkinlinks:
###排除test的应用
iflink['sourceInfo']['applicationName'].startswith('test'):
continue
#应用名称、应用类型、下游应用名称、下游应用类型、应用节点数、下游应用节点数、总请求数、错误请求数、慢请求数(本应用到下一个应用的数量)
application=link['sourceInfo']['applicationName']
serviceType=link['sourceInfo']['serviceType']
to_application=link['targetInfo']['applicationName']
to_serviceType=link['targetInfo']['serviceType']
agents=len(link.get('fromAgent',''))
to_agents=len(link.get('toAgent',''))
totalCount=link['totalCount']
errorCount=link['errorCount']
slowCount=link['slowCount']
sql="""
REPLACEintoapplication_server_map(application,serviceType,
agents,to_application,to_serviceType,to_agents,totalCount,
errorCount,slowCount,update_time,from_time,to_time)
VALUES("{}","{}",{},"{}","{}",{},{},{},{},"{}","{}",
"{}")""".format(
application,serviceType,agents,to_application,
to_serviceType,to_agents,totalCount,errorCount,
slowCount,update_time,From_Time,To_Time)
self.db.db_execute(sql)
defupdate_app(self):
"""更新application
"""
appdict=self.get_applications()
apps=appdict.get("applicationList")
update_time=time.strftime('%Y-%m-%d%H:%M:%S',time.localtime(time.time()))
forappinapps:
ifapp['applicationName'].startswith('test'):
continue
agents,agentlists=self.getAgentList(app['applicationName'])
sql="""
REPLACEintoapplication_list(application_name,
service_type,code,agents,agentlists,update_time)
VALUES("{}","{}",{},{},'{}',"{}");""".format(
app['applicationName'],app['serviceType'],
app['code'],agents,agentlists,update_time)
self.db.db_execute(sql)
returnTrue
defupdate_all_servermaps(self):
"""更新所有应用数
"""
appdict=self.get_applications()
apps=appdict.get("applicationList")
forappinapps:
self.update_servermap(app['applicationName'],serviceType=app['serviceType'])
###删除7天前数据
Del_Time=datetime.datetime.now()+datetime.timedelta(days=-7)
sql="""deletefromapplication_server_mapwhereupdate_time<="{}"
""".format(Del_Time)
self.db.db_execute(sql)
returnTrue
defconnect_db():
"""建立SQL连接
"""
mydb=db.MyDB(
host="rm-*****.mysql.rds.aliyuncs.com",
user="user",
passwd="passwd",
db="pinpoint"
)
mydb.db_connect()
mydb.db_cursor()
returnmydb
defmain():
db=connect_db()
pp=PinPoint(db)
pp.update_app()
pp.update_all_servermaps()
db.db_close()
if__name__=='__main__':
main()
附sql语句
CREATETABLE`application_list`( `application_name`varchar(32)NOTNULL, `service_type`varchar(32)DEFAULTNULLCOMMENT'服务类型', `code`int(11)DEFAULTNULLCOMMENT'服务类型代码', `agents`int(11)DEFAULTNULLCOMMENT'agent个数', `agentlists`varchar(256)DEFAULTNULLCOMMENT'agentlist', `update_time`datetimeDEFAULTNULLONUPDATECURRENT_TIMESTAMPCOMMENT'更新时间', PRIMARYKEY(`application_name`), UNIQUEKEY`Unique_App`(`application_name`)USINGBTREE )ENGINE=InnoDBDEFAULTCHARSET=utf8COMMENT='pinpointapplist' CREATETABLE`application_server_map`( `application`varchar(32)NOTNULLCOMMENT'应用名称', `serviceType`varchar(8)NOTNULL, `agents`int(2)NOTNULLCOMMENT'agent个数', `to_application`varchar(32)NOTNULLCOMMENT'下游服务名称', `to_serviceType`varchar(32)DEFAULTNULLCOMMENT'下游服务类型', `to_agents`int(2)DEFAULTNULLCOMMENT'下游服务agent数量', `totalCount`int(8)DEFAULTNULLCOMMENT'总请求数', `errorCount`int(8)DEFAULTNULL, `slowCount`int(8)DEFAULTNULL, `update_time`datetimeNOTNULLONUPDATECURRENT_TIMESTAMP, `from_time`datetimeDEFAULTNULL, `to_time`datetimeDEFAULTNULL, PRIMARYKEY(`application`,`to_application`), UNIQUEKEY`Unique_AppMap`(`application`,`to_application`)USINGBTREE )ENGINE=InnoDBDEFAULTCHARSET=utf8COMMENT='应用链路数据'
以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持毛票票。
声明：本文内容来源于网络，版权归原作者所有，内容由互联网用户自发贡献自行上传，本网站不拥有所有权，未作人工编辑处理，也不承担相关法律责任。如果您发现有涉嫌版权的内容，欢迎发送邮件至：czq8825#qq.com（发邮件时，请将#更换为@）进行举报，并提供相关证据，一经查实，本站将立刻删除涉嫌侵权内容。