结束提交

main.py

 from postgre.utils import util as pysql
 from yearly_report_analysis import year
 import re
+import math
 
 
 class Database():
   html = ''
   res = ''
   count = 10
-  start = 120
+  start = 0
+  length = 0
   year = ''
   country = ''
   type = ''
@@ -81,20 +83,27 @@ class Database():
   }  # 6
 
   def __init__(self):
+    self.get_database_table_length()
     self.get_disease_list()
-    self.get_html()
+    # self.get_html()
     self.ana_to_json()
 
+  def get_database_table_length(self):
+    data = pysql.selectData(['count(1)'], 'oie_yearlyreport_new')
+    self.length = data[0]['count(1)']
+
   def get_html(self):
+    self.print_and_log('开始获取' + str(self.start) + '后' + str(self.count) + '条')
     select_res = pysql.selectData(('country_code', 'year', 'type', 'json_text', 'id'), 'oie_yearlyreport_new', False,
                                   (self.count, self.start))
     self.select_res = select_res
+    self.print_and_log(str(self.start) + '后' + str(self.count) + '条获取成功，开始解析')
 
   def ana_to_json(self):
     self.res = year.analysis(self.html)
 
   def analyse_ctrl(self):
-    for i in range(10, 20):
+    for i in range(0, math.ceil(self.length / self.count)):
       self.get_html()
       for j in self.select_res:
         self.year = j['year']
@@ -112,6 +121,9 @@ class Database():
         if i[j]['no'] == 1:  # summary
           table_1 = self.handle_a_table_1(i[j])
           self.insert_into_1(table_1)
+      break
+    self.print_and_log('第' + str(self.start) + '条分析完成  年份：' + str(self.year) + ' 国家：' + self.country + ' 类型:' + self.type)
+
           # print(table_1)
         # elif i[j]['no'] == 2:  # not_report
           # table_2 = self.handle_a_table_2(i[j]['tables'])
@@ -165,10 +177,6 @@ class Database():
         # elif i[j]['no'] == 16:
           # table_16 = self.handle_a_table_16(i[j])
           # print(table_16)
-    with open('ana.log', 'a') as fileReader:
-      fileReader.write('第' + str(self.start) + '条分析完成  年份：' + str(self.year) + ' 国家：' + self.country + ' 类型:' + self.type)
-    print('第' + str(self.start) + '条分析完成  年份：' + str(self.year) + ' 国家：' + self.country + ' 类型:' + self.type)
-
   def handle_a_table_1(self, table):
     auto = [0, 1, 2]
     return self.ana_table_to_ok(table, auto)
@@ -384,7 +392,7 @@ class Database():
     else:
       pysql.insertData({'oiednameen': disease_name}, 'oie_disease')
       self.get_disease_list()
-      self.get_disease_standard_id(disease_name)
+      return self.get_disease_standard_id(disease_name)
 
   def get_standard_disease_name(self, disease_name):
     disease_name = disease_name.replace(' (Domestic and Wild)', '')
@@ -444,6 +452,12 @@ class Database():
       "'", '')
     return k
 
+  def print_and_log(self, text):
+    print(text)
+    with open('ana.log', 'a') as file_write:
+      file_write.write(text)
+      file_write.write('\n')
+
 
 year_report = Database()
 year_report.analyse_ctrl()

postgre/utils.py

@@ -122,13 +122,11 @@ tableInfos = {
 class psqlUtil(object):
 
     def open(self):
-        # if not hasattr(self, "conn"):
         # 连接到一个给定的数据库
-        self.conn = psycopg2.connect(database="animal_new", user="animal", password="animal@9589", host="172.17.1.61",
-                                 port="5432")
+        self.conn = psycopg2.connect(database="animal_new", user="animal", password="animal@9589", host="172.17.1.61", port="5432")
+        # self.conn = psycopg2.connect(database="animal_disease", user="postgres", password="postgres", host="39.104.109.13", port="5432")
         # 建立游标，用来执行数据库操作
         self.cursor = self.conn.cursor()
-        # print("__init__.py", self.conn)
 
     def close(self):
         # 关闭游标
@@ -147,11 +145,10 @@ class psqlUtil(object):
             sql += getValueFromInfo(key, value, tableName) + ","
         sql = sql[0:-1:1]
         sql += ")"
-        # print(sql)
         self.cursor.execute(sql)
         self.conn.commit()
 
-    def selectData(self, fields, tableName, where = False,limit = False):
+    def selectData(self, fields, tableName, where=False, limit=False):
         self.open()
         sql = "SELECT "
         for f in fields: