Merge branch 'master' of git.craftbroec.nl:VBOO/FAGB

.gitignore

@@ -1,3 +1,4 @@
 .venv/
 .env
 __pycache__
+*.html

app/__init__.py

@@ -14,12 +14,17 @@ from app.models import Base, State, Region, Player
 load_dotenv()
 
 # database
-engine = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
-Session = sessionmaker(bind=engine)
+ENGINE = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
+SESSION = sessionmaker(bind=ENGINE)
 
 # scheduler
-scheduler = BackgroundScheduler()
-scheduler.start()
+SCHEDULER = BackgroundScheduler(
+    daemon=True,
+    job_defaults={
+        'misfire_grace_time': 900
+    },
+)
+SCHEDULER.start()
 
 # logging
 logging.basicConfig(

app/__main__.py

@@ -2,37 +2,36 @@
 
 import time
 
-from app import scheduler, LOGGER
+from app import SCHEDULER, LOGGER
 from app.api import get_factories
-from app.database import get_state, save_factories
+from app.database import get_state, get_regions, save_factories
 
 
 def print_factories(factories):
     """Print professors"""
     for factory in factories:
-        print('{:30} {:24} {:2} {:3} {:3}'.format(
-            factory['name'],
-            factory['region_name'],
-            factory['resource_type'],
-            factory['level'],
-            factory['workers'],
-        ))
+        print(factory)
 
 def job_update_factories(state_id):
     """Update factories"""
     LOGGER.info('Run update factories for state "%s"', state_id)
     state = get_state(state_id)
-    LOGGER.info('"%s": get factories', state.name)
-    factories = get_factories(state.id)
-    LOGGER.info('"%s": "%s" factories', state.name, len(factories))
-    # print_factories(factories)
-    save_factories(state.id, factories)
+    LOGGER.info('"%s": get regions', state.name)
+    regions = get_regions(state.id)
+    factory_list = []
+    for region in regions:
+        LOGGER.info('"%s": get factories', region.name)
+        factories = get_factories(region.id)
+        LOGGER.info('"%s": "%s" factories', region.name, len(factories))
+        factory_list += factories
+
+    save_factories(state_id, factory_list)
     LOGGER.info('"%s": done saving factories', state.name)
 
 
 def add_update_factories(state_id):
     """Add jobs"""
-    scheduler.add_job(
+    SCHEDULER.add_job(
         job_update_factories,
         'cron',
         args=[state_id],
@@ -42,7 +41,7 @@ def add_update_factories(state_id):
     )
 
 if __name__ == '__main__':
-    # job_update_factories(2788)
+    job_update_factories(2788)
 
     # jobs
     # Verenigde Nederlanden
@@ -57,4 +56,5 @@ if __name__ == '__main__':
             time.sleep(100)
     except KeyboardInterrupt:
         LOGGER.info('Exiting application')
+        SCHEDULER.shutdown()
         exit()

app/api.py

@@ -17,50 +17,45 @@ TYPES = {
     'diamond': 15,
 }
 
-def get_factories(state_id):
+def get_factories(region_id):
     """Get factories from state"""
     # return read_factories()
-    return download_factories(state_id)
+    return download_factories(region_id)
 
 def read_factories():
     """Read factories file"""
-    with open('factories.html') as file:
-        factories, more = parse_factories(file)
+    with open('factories_4001.html') as file:
+        factories, more = parse_factories(file, 4001)
         return factories
 
-def download_factories(state_id):
+def download_factories(region_id):
     """Download the factories"""
     factories = []
     more = True
     page = 0
     while more:
         response = requests.get(
-            '{}factory/state/{}/0/0/{}'.format(BASE_URL, state_id, page*25),
+            '{}factory/search/{}/0/0/{}'.format(BASE_URL, region_id, page*25),
             headers=HEADERS
         )
-        tmp_factories, more = parse_factories(response.text)
+        tmp_factories, more = parse_factories(response.text, region_id)
         factories = factories + tmp_factories
         page += 1
     return factories
 
-def parse_factories(html):
+def parse_factories(html, region_id):
     """Parse html return factories"""
     soup = BeautifulSoup(html, 'html.parser')
     factories_tree = soup.find_all(class_='list_link')
     factories = []
     for factory_tree in factories_tree:
         columns = factory_tree.find_all('td')
-        if columns[1].contents[4].name == 'span':
-            resource_type = TYPES[columns[1].contents[4]['class'][0]]
-        else:
-            resource_type = None
         factories.append({
-            'id': factory_tree['user'],
+            'region_id': region_id,
+            'id': int(factory_tree['user']),
             'name': columns[1].contents[0].strip(),
-            'resource_type': resource_type,
-            'region_name': columns[1].contents[2],
-            'level': columns[2].string,
-            'workers': re.sub(r'\/[0-9]*$', '', columns[3].string),
+            'level': int(columns[2].string),
+            'workers': int(re.sub(r'\/[0-9]*$', '', columns[3].string)),
             'wage': int(columns[4].string.replace('%', '')),
             'experience': int(columns[5].string),
         })

app/database.py

@@ -2,20 +2,34 @@
 
 from datetime import datetime
 
-from app import Session
-from app.models import State, Region, Factory, FactoryTrack, FactoryStat, FactoryLocation
+from app import SESSION
+from app.models import State, Region, StateRegion, \
+    Factory, FactoryTrack, FactoryStat, FactoryLocation
 
 
 def get_state(state_id):
     """Get regions from state"""
-    session = Session()
+    session = SESSION()
     state = session.query(State).get(state_id)
     session.close()
     return state
 
+def get_regions(state_id):
+    """Get region from state"""
+    session = SESSION()
+    state_regions = session.query(StateRegion) \
+        .filter(StateRegion.state_id == state_id) \
+        .filter(StateRegion.until_date_time == None) \
+        .all()
+    regions = []
+    for state_region in state_regions:
+        regions.append(state_region.region)
+    session.close()
+    return regions
+
 def save_factories(state_id, factories):
     """Save factories to database"""
-    session = Session()
+    session = SESSION()
     session.close()
 
     factory_track = FactoryTrack()
@@ -40,9 +54,9 @@ def save_factories(state_id, factories):
             .filter(FactoryLocation.factory_id == factory.id) \
             .filter(FactoryLocation.until_date_time == None).first()
 
-        if not current_location or current_location.region.name != factory_dict['region_name']:
+        if not current_location or current_location.region_id != factory_dict['region_id']:
             region = session.query(Region) \
-                .filter(Region.name == factory_dict['region_name']).first()
+                .filter(Region.id == factory_dict['region_id']).first()
             factory_location = FactoryLocation()
             factory_location.factory_id = factory.id
             factory_location.region_id = region.id
@@ -60,6 +74,5 @@ def save_factory(session, factory_dict):
     factory = Factory()
     factory.id = factory_dict['id']
     factory.name = factory_dict['name']
-    factory.resource_type = factory_dict['resource_type']
     session.add(factory)
     return factory

app/models.py

@@ -79,6 +79,20 @@ class Region(Base):
     name = Column(String)
 
 
+class StateRegion(Base):
+    """Model for state region"""
+    __tablename__ = 'state_region'
+    state_id = Column(Integer, ForeignKey('state.id'), primary_key=True)
+    from_date_time = Column(DateTime, primary_key=True)
+    until_date_time = Column(DateTime)
+
+    region_id = Column(Integer, ForeignKey('region.id'), primary_key=True)
+    region = relationship(
+        'Region',
+        backref=backref('state_regions', lazy='dynamic')
+    )
+
+
 class Player(Base):
     """Model for player"""
     __tablename__ = 'player'