ok

.gitignore

+.idea/
+venv/
+__pycache__/
+*.html
+*.xml
\ No newline at end of file

.idea/dataSources.local.xml

@@ -8,7 +8,16 @@
       <case-sensitivity plain-identifiers="lower" quoted-identifiers="exact" />
       <secret-storage>master_key</secret-storage>
       <user-name>postgres</user-name>
-      <introspection-schemas>animal_new:@,public</introspection-schemas>
+      <schema-mapping>
+        <introspection-scope>
+          <node kind="database" qname="animal_new">
+            <node kind="schema">
+              <name qname="@" />
+              <name qname="public" />
+            </node>
+          </node>
+        </introspection-scope>
+      </schema-mapping>
     </data-source>
   </component>
 </project>
\ No newline at end of file

.idea/dataSources/16830a47-a264-4867-9152-47f09ec57145.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <dataSource name="postgres@39.104.109.13">
-  <database-model serializer="dbm" dbms="POSTGRES" family-id="POSTGRES" format-version="4.15">
+  <database-model serializer="dbm" dbms="POSTGRES" family-id="POSTGRES" format-version="4.17">
     <root id="1">
       <ServerVersion>9.5.12</ServerVersion>
     </root>
     <database id="2" parent="1" name="agricultural_disasters">
       <ObjectId>3756219</ObjectId>
-      <StateNumber>18154165</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="3" parent="1" name="animal">
       <ObjectId>16394</ObjectId>
-      <StateNumber>618</StateNumber>
       <Owner>animal</Owner>
     </database>
     <database id="4" parent="1" name="animal_new">
       <ObjectId>3195021</ObjectId>
-      <StateNumber>10491014</StateNumber>
       <Owner>postgres</Owner>
       <IntrospectionStateNumber>19114361</IntrospectionStateNumber>
-      <Visible>1</Visible>
-      <Relations>sequence|table|3196649|3196651|1
-sequence|table|3197208|3196882|1
-sequence|table|3197210|3196892|1
-sequence|table|3197212|3196898|1
-sequence|table|3197214|3196966|1
-sequence|table|3197216|3196973|1
-sequence|table|3197218|3196976|1
-sequence|table|3197220|3196985|1
-sequence|table|3197222|3196988|1
-sequence|table|3197224|3196994|1
-sequence|table|3197226|3196998|1
-sequence|table|3197228|3197001|1
-sequence|table|3197230|3197010|1
-sequence|table|3197234|3197030|1
-sequence|table|3197236|3197033|1
-sequence|table|3197238|3197043|1
+      <Relations>sequence|table|3197214|3196966|1
+sequence|table|3197987|3197980|1
+sequence|table|3197960|3197954|1
+sequence|table|3197579|3197572|1
 sequence|table|3197240|3197093|1
 sequence|table|3197242|3197101|1
-sequence|table|3197244|3197109|1
-sequence|table|3197246|3197112|1
-sequence|table|3197248|3197115|1
-sequence|table|3197570|3197564|1
-sequence|table|3197579|3197572|1
-sequence|table|3197597|3197591|1
-sequence|table|3197666|3197655|1
 sequence|table|3197675|3197668|11
-sequence|table|3197685|3197677|1
-sequence|table|3197687|3197677|8
 sequence|table|3197703|3197696|13
-sequence|table|3197728|3197543|10
-sequence|table|3197737|3197730|10
-sequence|table|3197739|3197705|1
-sequence|table|3197748|3197741|10
-sequence|table|3197757|3197750|11
-sequence|table|3197774|3197767|10
+sequence|table|3197224|3196994|1
 sequence|table|3197783|3197776|8
+sequence|table|3966714|3966655|11
+sequence|table|3197244|3197109|1
+sequence|table|3197226|3196998|1
+sequence|table|3196649|3196651|1
+sequence|table|3197238|3197043|1
+sequence|table|3197230|3197010|1
+sequence|table|3197978|3197971|1
+sequence|table|3197993|3197989|1
+sequence|table|3197939|3197933|1
+sequence|table|3197895|3197889|6
+sequence|table|3197216|3196973|1
+sequence|table|3197685|3197677|1
+sequence|table|3197666|3197655|1
 sequence|table|3197791|3197785|1
-sequence|table|3197800|3197793|1
-sequence|table|3197809|3197802|1
+sequence|table|3197234|3197030|1
+sequence|table|3979184|3197720|12
+sequence|table|3197687|3197677|8
+sequence|table|3197220|3196985|1
+sequence|table|3197774|3197767|10
+sequence|table|3197208|3196882|1
+sequence|table|3197246|3197112|1
+sequence|table|3197228|3197001|1
+sequence|table|3197212|3196898|1
+sequence|table|3197739|3197705|1
 sequence|table|3197811|3197616|1
-sequence|table|3197895|3197889|6
-sequence|table|3197939|3197933|1
-sequence|table|3197960|3197954|1
 sequence|table|3197969|3197962|1
-sequence|table|3197978|3197971|1
-sequence|table|3197987|3197980|1
-sequence|table|3197993|3197989|1
-sequence|table|3197999|3197995|1
-sequence|table|3198008|3198001|1
+sequence|table|3197218|3196976|1
+sequence|table|3197570|3197564|1
+sequence|table|3718020|3718022|1
+sequence|table|3197809|3197802|1
+sequence|table|3197222|3196988|1
+sequence|table|3197236|3197033|1
+sequence|table|3197737|3197730|10
+sequence|table|3197757|3197750|11
+sequence|table|3197248|3197115|1
+sequence|table|4059906|4045657|21
 sequence|table|3490841|3490821|3
-sequence|table|3504534|3504536|1
 sequence|table|3582263|3582265|1
+sequence|table|3198008|3198001|1
+sequence|table|3197999|3197995|1
+sequence|table|3197728|3197543|10
+sequence|table|3504534|3504536|1
+sequence|table|3197800|3197793|1
+sequence|table|3197597|3197591|1
 sequence|table|3671556|3619949|6
-sequence|table|3718020|3718022|1
-sequence|table|3966714|3966655|11
-sequence|table|3979184|3197720|12
-sequence|table|4059906|4045657|21
+sequence|table|3197748|3197741|10
+sequence|table|3197210|3196892|1
 </Relations>
     </database>
     <database id="5" parent="1" name="dataCheck">
       <ObjectId>2323817</ObjectId>
-      <StateNumber>4732876</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="6" parent="1" name="disease">
       <ObjectId>3860811</ObjectId>
-      <StateNumber>18275196</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="7" parent="1" name="examine">
       <ObjectId>2248088</ObjectId>
-      <StateNumber>4393023</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="8" parent="1" name="field">
       <ObjectId>3189354</ObjectId>
       <Comment>产地监测</Comment>
-      <StateNumber>10477961</StateNumber>
       <Owner>field</Owner>
     </database>
     <database id="9" parent="1" name="field_web">
       <ObjectId>4014730</ObjectId>
-      <StateNumber>18457649</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="10" parent="1" name="fish">
       <ObjectId>2184194</ObjectId>
-      <StateNumber>2627635</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="11" parent="1" name="fisheries-analysis">
       <ObjectId>2377581</ObjectId>
-      <StateNumber>5869612</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="12" parent="1" name="fishery_diease">
       <ObjectId>3624236</ObjectId>
-      <StateNumber>11448439</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="13" parent="1" name="new_field">
       <ObjectId>4014645</ObjectId>
-      <StateNumber>18457528</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="14" parent="1" name="nyb">
       <ObjectId>2670430</ObjectId>
-      <StateNumber>5869729</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="15" parent="1" name="oie">
       <ObjectId>4051704</ObjectId>
-      <StateNumber>19101491</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="16" parent="1" name="oie_disease">
       <ObjectId>4014994</ObjectId>
-      <StateNumber>18457895</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="17" parent="1" name="pollsurvey">
       <ObjectId>32132</ObjectId>
-      <StateNumber>164315</StateNumber>
       <Owner>pollsurvey</Owner>
     </database>
     <database id="18" parent="1" name="postgres">
       <ObjectId>12373</ObjectId>
       <Comment>default administrative connection database</Comment>
-      <StateNumber>614</StateNumber>
       <Owner>postgres</Owner>
       <Current>1</Current>
     </database>
     <database id="19" parent="1" name="sl">
       <ObjectId>2329843</ObjectId>
-      <StateNumber>4866704</StateNumber>
       <Owner>animal</Owner>
     </database>
     <database id="20" parent="1" name="test">
       <ObjectId>2323899</ObjectId>
-      <StateNumber>4732967</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="21" parent="1" name="test_disease">
       <ObjectId>4040592</ObjectId>
-      <StateNumber>18658508</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <database id="22" parent="1" name="wupu">
       <ObjectId>2184825</ObjectId>
-      <StateNumber>2630986</StateNumber>
       <Owner>postgres</Owner>
     </database>
     <role id="23" parent="1" name="animal">
       <ObjectId>16393</ObjectId>
-      <Inherit>1</Inherit>
       <CanLogin>1</CanLogin>
-      <ConnectionLimit>-1</ConnectionLimit>
       <ValidUntil>infinity</ValidUntil>
     </role>
     <role id="24" parent="1" name="field">
       <ObjectId>3189353</ObjectId>
       <SuperRole>1</SuperRole>
-      <Inherit>1</Inherit>
       <CreateRole>1</CreateRole>
       <CreateDb>1</CreateDb>
       <CanLogin>1</CanLogin>
-      <ConnectionLimit>-1</ConnectionLimit>
     </role>
     <role id="25" parent="1" name="pollsurvey">
       <ObjectId>27213</ObjectId>
       <SuperRole>1</SuperRole>
-      <Inherit>1</Inherit>
       <CanLogin>1</CanLogin>
-      <ConnectionLimit>-1</ConnectionLimit>
       <ValidUntil>infinity</ValidUntil>
       <BypassRls>1</BypassRls>
     </role>
     <role id="26" parent="1" name="postgres">
       <ObjectId>10</ObjectId>
       <SuperRole>1</SuperRole>
-      <Inherit>1</Inherit>
       <CreateRole>1</CreateRole>
       <CreateDb>1</CreateDb>
       <CanLogin>1</CanLogin>
-      <ConnectionLimit>-1</ConnectionLimit>
       <Replication>1</Replication>
       <BypassRls>1</BypassRls>
     </role>
     <role id="27" parent="1" name="zhijian">
       <ObjectId>2758451</ObjectId>
-      <Inherit>1</Inherit>
       <CanLogin>1</CanLogin>
-      <ConnectionLimit>-1</ConnectionLimit>
     </role>
     <schema id="28" parent="4" name="information_schema">
       <ObjectId>12081</ObjectId>
@@ -253,7 +221,6 @@ sequence|table|4059906|4045657|21
       <Owner>postgres</Owner>
       <IntrospectionStateNumber>19114362</IntrospectionStateNumber>
       <Current>1</Current>
-      <Visible>1</Visible>
     </schema>
     <schema id="35" parent="4" name="topology">
       <ObjectId>3195039</ObjectId>

.idea/misc.xml

@@ -3,5 +3,5 @@
   <component name="JavaScriptSettings">
     <option name="languageLevel" value="ES6" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (python-project)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (oie-yearly-report)" project-jdk-type="Python SDK" />
 </project>
\ No newline at end of file

.idea/oie-yearly-report.iml

 <?xml version="1.0" encoding="UTF-8"?>
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.6 (oie-yearly-report)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TestRunnerService">

main.py

@@ -50,7 +50,6 @@ def sub_select(length, disease_dict):
 
 
 def handle_all_analysis(a_html, disease_dict, year_report_id, country_code, year, type):
-    print(year_report_id)
     if a_html == []:
         with open('err.log', 'a', encoding='utf-8') as file:
             file.writelines(str([a_html, year_report_id, country_code, year, type]) + '\n')
@@ -73,7 +72,7 @@ def handle_all_analysis(a_html, disease_dict, year_report_id, country_code, year
             continue
         else:
             insert_into_summary('oie_year_report_summary', tmp)
-    table_3 = handle_a_table_3(table_3)
+    # table_3 = handle_a_table_3(table_3)
     return True
 
 

postgre/utils.py

@@ -108,6 +108,14 @@ tableInfos = {
     "type": "string",
     "year_report_id": "int",
     "id": "int"
+  },
+  'non_listed_col_dict': {
+    'disease': 'string',
+    'disease_id': 'int',
+    'date_of_last_occurrence': 'string',
+    'species': 'string',
+    'control_measures': 'string',
+    'official_vaccination': 'string'
   }
 }
 

test.py

@@ -2,11 +2,12 @@ from postgre.utils import util as pysql
 from yearly_report_analysis import year
 import re
 
+
 class Database():
   html = ''
   res = ''
   count = 10
-  start = 1
+  start = 120
   year = ''
   country = ''
   type = ''
@@ -70,7 +71,14 @@ class Database():
     "Killed and disposed of": "killed_and_disposed_of",
     "Slaughtered": "slaughtered",
     "Vaccination in response to the outbreak(s)": "vaccination_in_response_to_the_outbreak"
-  }
+  }  # 5
+  non_listed_col_dict = {
+    'Disease': 'disease',
+    'Date of last occurrence': 'date_of_last_occurrence',
+    'Species': 'species',
+    'Control Measures': 'control_measures',
+    'Official vaccination': 'official_vaccination'
+  }  # 6
 
   def __init__(self):
     self.get_disease_list()
@@ -78,14 +86,15 @@ class Database():
     self.ana_to_json()
 
   def get_html(self):
-    select_res = pysql.selectData(('country_code', 'year', 'type', 'json_text', 'id'), 'oie_yearlyreport_new', False, (self.count, self.start))
+    select_res = pysql.selectData(('country_code', 'year', 'type', 'json_text', 'id'), 'oie_yearlyreport_new', False,
+                                  (self.count, self.start))
     self.select_res = select_res
 
   def ana_to_json(self):
     self.res = year.analysis(self.html)
 
   def analyse_ctrl(self):
-    for i in range(1, 10):
+    for i in range(10, 20):
       self.get_html()
       for j in self.select_res:
         self.year = j['year']
@@ -100,15 +109,15 @@ class Database():
     self.start = self.start + 1
     for i in self.res:
       for j in i:
-        if i[j]['no'] == 1: # summary
+        if i[j]['no'] == 1:  # summary
           table_1 = self.handle_a_table_1(i[j])
-          # self.insert_into_1(table_1)
+          self.insert_into_1(table_1)
           # print(table_1)
-        elif i[j]['no'] == 2: # not_report
+        elif i[j]['no'] == 2:  # not_report
           table_2 = self.handle_a_table_2(i[j]['tables'])
           # self.insert_into_2(table_2)
           # print(table_2)
-        elif i[j]['no'] == 3: # detail
+        elif i[j]['no'] == 3:  # detail
           table_3 = self.handle_a_table_3(i[j]['tables'])
           # self.insert_into_3(table_3)
           # print(table_3)
@@ -118,11 +127,14 @@ class Database():
           # print(table_4)
         elif i[j]['no'] == 5:
           table_5 = self.handle_a_table_5(i[j])
-          if table_5:
-            self.insert_into_5(table_5)
+          # if table_5:
+          # self.insert_into_5(table_5)
           # print(table_5)
         elif i[j]['no'] == 6:
-          table_6 = self.handle_a_table_6(i[j])
+          pass
+          # table_6 = self.handle_a_table_6(i[j])
+          # if table_6:
+          #   self.insert_into_6(table_6)
           # print(table_6)
         elif i[j]['no'] == 7:
           table_7 = self.handle_a_table_7(i[j]['tables'])
@@ -266,7 +278,6 @@ class Database():
         }
         self.insert_to_table(tmp, "oie_year_report_unreported", "表4已存在")
 
-
   def handle_a_table_5(self, table):
     auto = [0, 1, 2]
     if table['type'] == 'table':
@@ -284,7 +295,6 @@ class Database():
         tmp[ths[key]] = val
       self.insert_to_table(tmp, "oie_year_report_summary_non", '表5数据已存在')
 
-
   def handle_a_table_6(self, table):
     if table['tables']:
       ok_table = []
@@ -292,6 +302,10 @@ class Database():
         for type, item in i.items():
           a_ok_table = self.ana_table_to_ok(item)
           ok_table.append({type: a_ok_table})
+      return ok_table
+
+  def insert_into_6(self, table):
+    print(table)
 
   def handle_a_table_7(self, table):
     auto = [0]
@@ -307,7 +321,7 @@ class Database():
         ok_table.append(a_ok_table)
     return ok_table
 
-  def handle_a_table_8(self,  table):
+  def handle_a_table_8(self, table):
     res = self.handle_a_table_4(table['tables'])
     pass
 
@@ -317,7 +331,8 @@ class Database():
 
   def handle_a_table_10(self, table):
     auto = [0]
-    res = self.ana_table_to_ok(table, auto)
+    if 'th' in table:
+      res = self.ana_table_to_ok(table, auto)
 
   def handle_a_table_11(self, table):
     tables = []
@@ -393,7 +408,7 @@ class Database():
     else:
       pysql.insertData(data, table_name)
 
-  def ana_table_to_ok(self, a_table, auto_col = [0]):
+  def ana_table_to_ok(self, a_table, auto_col=[0]):
     auto_data = {}
     th = a_table['th']
     ths = []
@@ -423,8 +438,11 @@ class Database():
     return {'th': ths, 'td': tds}
 
   def dict_key(self, data):
-    k = str(data.keys()).replace("dict_keys", "").replace("(", "").replace(")", "").replace("[", "").replace("]", "").replace("'", '')
+    k = str(data.keys()).replace("dict_keys", "").replace("(", "").replace(")", "").replace("[", "").replace("]",
+                                                                                                             "").replace(
+      "'", '')
     return k
 
+
 year_report = Database()
-year_report.analyse_ctrl()
\ No newline at end of file
+year_report.analyse_ctrl()