import json
import time
import requests
class exportEsData():
size = 10000
def __init__(self, url,index,type):
self.url = url+"/"+index+"/"+type+"/_search"
self.index = index
self.type = type
def exportData(self):
print("export data begin...")
begin = time.time()
print(self.url)
msg = requests.get(self.url).text
print(msg)
obj = json.loads(msg)
num = obj["hits"]["total"]
start = 0
end = num/self.size+1
while(start<end):
msg = requests.get(self.url+"?from="+str(start*self.size)+"&size="+str(self.size)).text
self.writeFile(msg)
start=start+1
print("export data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")
def writeFile(self,msg):
obj = json.loads(msg)
#print(obj)
vals = obj["hits"]["hits"]
try:
f = open("/Users/haonanzhang/"+self.index+"_"+self.type+".json","a")
for val in vals:
a = json.dumps(val["_source"])
f.write(a+"\n")
finally:
f.flush()
f.close()
if __name__ == '__main__':
exportEsData("url:port,index,type).exportData()