Fix issues with downloading

app/__main__.py

@@ -10,13 +10,7 @@ from app.database import get_state, get_regions, save_factories
 def print_factories(factories):
     """Print professors"""
     for factory in factories:
-        print('{:30} {:24} {:2} {:3} {:3}'.format(
-            factory['name'],
-            factory['region_name'],
-            factory['resource_type'],
-            factory['level'],
-            factory['workers'],
-        ))
+        print(factory)
 
 def job_update_factories(state_id):
     """Update factories"""
@@ -24,12 +18,15 @@ def job_update_factories(state_id):
     state = get_state(state_id)
     LOGGER.info('"%s": get regions', state.name)
     regions = get_regions(state.id)
+    factory_list = []
     for region in regions:
         LOGGER.info('"%s": get factories', region.name)
         factories = get_factories(region.id)
         LOGGER.info('"%s": "%s" factories', region.name, len(factories))
+        factory_list += factories
         print_factories(factories)
-        # save_factories(region.id, factories)
+
+    save_factories(state_id, factory_list)
     LOGGER.info('"%s": done saving factories', state.name)
 
 

app/api.py

@@ -24,8 +24,8 @@ def get_factories(region_id):
 
 def read_factories():
     """Read factories file"""
-    with open('factories.html') as file:
-        factories, more = parse_factories(file)
+    with open('factories_4001.html') as file:
+        factories, more = parse_factories(file, 4001)
         return factories
 
 def download_factories(region_id):
@@ -38,29 +38,24 @@ def download_factories(region_id):
             '{}factory/search/{}/0/0/{}'.format(BASE_URL, region_id, page*25),
             headers=HEADERS
         )
-        tmp_factories, more = parse_factories(response.text)
+        tmp_factories, more = parse_factories(response.text, region_id)
         factories = factories + tmp_factories
         page += 1
     return factories
 
-def parse_factories(html):
+def parse_factories(html, region_id):
     """Parse html return factories"""
     soup = BeautifulSoup(html, 'html.parser')
     factories_tree = soup.find_all(class_='list_link')
     factories = []
     for factory_tree in factories_tree:
         columns = factory_tree.find_all('td')
-        if columns[1].contents[4].name == 'span':
-            resource_type = TYPES[columns[1].contents[4]['class'][0]]
-        else:
-            resource_type = None
         factories.append({
-            'id': factory_tree['user'],
+            'region_id': region_id,
+            'id': int(factory_tree['user']),
             'name': columns[1].contents[0].strip(),
-            'resource_type': resource_type,
-            'region_name': columns[1].contents[2],
-            'level': columns[2].string,
-            'workers': re.sub(r'\/[0-9]*$', '', columns[3].string),
+            'level': int(columns[2].string),
+            'workers': int(re.sub(r'\/[0-9]*$', '', columns[3].string)),
             'wage': int(columns[4].string.replace('%', '')),
             'experience': int(columns[5].string),
         })

app/database.py

@@ -3,7 +3,8 @@
 from datetime import datetime
 
 from app import SESSION
-from app.models import State, Region, Factory, FactoryTrack, FactoryStat, FactoryLocation
+from app.models import State, Region, StateRegion, \
+    Factory, FactoryTrack, FactoryStat, FactoryLocation
 
 
 def get_state(state_id):
@@ -16,8 +17,13 @@ def get_state(state_id):
 def get_regions(state_id):
     """Get region from state"""
     session = SESSION()
-    state = session.query(State).get(state_id)
-    regions = state.regions
+    state_regions = session.query(StateRegion) \
+        .filter(StateRegion.state_id == state_id) \
+        .filter(StateRegion.until_date_time == None) \
+        .all()
+    regions = []
+    for state_region in state_regions:
+        regions.append(state_region.region)
     session.close()
     return regions
 
@@ -48,7 +54,7 @@ def save_factories(state_id, factories):
             .filter(FactoryLocation.factory_id == factory.id) \
             .filter(FactoryLocation.until_date_time == None).first()
 
-        if not current_location or current_location.region.name != factory_dict['region_name']:
+        if not current_location or current_location.region_id != factory_dict['region_id']:
             region = session.query(Region) \
                 .filter(Region.name == factory_dict['region_name']).first()
             factory_location = FactoryLocation()

app/models.py

@@ -79,6 +79,20 @@ class Region(Base):
     name = Column(String)
 
 
+class StateRegion(Base):
+    """Model for state region"""
+    __tablename__ = 'state_region'
+    state_id = Column(Integer, ForeignKey('state.id'), primary_key=True)
+    from_date_time = Column(DateTime, primary_key=True)
+    until_date_time = Column(DateTime)
+
+    region_id = Column(Integer, ForeignKey('region.id'), primary_key=True)
+    region = relationship(
+        'Region',
+        backref=backref('state_regions', lazy='dynamic')
+    )
+
+
 class Player(Base):
     """Model for player"""
     __tablename__ = 'player'