Finished saving factories and tracks

app/__init__.py

@@ -16,14 +16,14 @@ load_dotenv()
 # database
 engine = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
 Session = sessionmaker(bind=engine)
-session = Session()
 
 # scheduler
 scheduler = BackgroundScheduler()
 scheduler.start()
 
+# logging
 logging.basicConfig(
-    format='%(name)s - %(levelname)s - %(message)s',
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
     level=logging.INFO
 )
 LOGGER = logging.getLogger(__name__)

app/__main__.py

@@ -2,7 +2,7 @@
 
 import time
 
-from app import scheduler, session, LOGGER
+from app import scheduler, LOGGER
 from app.api import get_factories
 from app.database import get_state, save_factories
 
@@ -10,9 +10,12 @@ from app.database import get_state, save_factories
 def print_factories(factories):
     """Print professors"""
     for factory in factories:
-        print('{:20} {:30}'.format(
-            factory['id'],
+        print('{:30} {:24} {:2} {:3} {:3}'.format(
             factory['name'],
+            factory['region_name'],
+            factory['resource_type'],
+            factory['level'],
+            factory['workers'],
         ))
 
 def job_update_factories(state_id):
@@ -23,7 +26,7 @@ def job_update_factories(state_id):
     factories = get_factories(state.id)
     LOGGER.info('"%s": "%s" factories', state.name, len(factories))
     print_factories(factories)
-    # save_factories(region.id, factories)
+    save_factories(state.id, factories)
     LOGGER.info('"%s": done saving factories', state.name)
 
 
@@ -40,7 +43,7 @@ def add_update_factories(state_id):
 
 if __name__ == '__main__':
     # jobs
-    # job_update_factories(2788)
+    job_update_factories(2788)
 
     # Verenigde Nederlanden
     add_update_factories(2788)
@@ -53,6 +56,5 @@ if __name__ == '__main__':
         while True:
             time.sleep(100)
     except KeyboardInterrupt:
-        print('Exiting application')
-        session.close()
+        LOGGER.info('Exiting application')
         exit()

app/api.py

@@ -9,20 +9,58 @@ from bs4 import BeautifulSoup
 from app import BASE_URL, HEADERS
 
 
+TYPES = {
+    'yellow': 1,
+    'oil': 2,
+    'ore': 5,
+    'uranium': 11,
+    'diamond': 15,
+}
+
 def get_factories(state_id):
     """Get factories from state"""
-    return download_factories(state_id)
+    return read_factories()
+    # return download_factories(state_id)
+
+def read_factories():
+    """Read factories file"""
+    with open('factories.html') as file:
+        factories, more = parse_factories(file)
+        return factories
 
 def download_factories(state_id):
-    """Download the players"""
-    return []
+    """Download the factories"""
+    factories = []
+    more = True
+    page = 0
+    while more:
+        response = requests.get(
+            '{}factory/state/{}/0/0/{}'.format(BASE_URL, state_id, page*25),
+            headers=HEADERS
+        )
+        tmp_factories, more = parse_factories(response.text)
+        factories = factories + tmp_factories
+        page += 1
+    return factories
 
 def parse_factories(html):
     """Parse html return factories"""
     soup = BeautifulSoup(html, 'html.parser')
     factories_tree = soup.find_all(class_='list_link')
     factories = []
-    return factories
+    for factory_tree in factories_tree:
+        columns = factory_tree.find_all('td')
+        factories.append({
+            'id': factory_tree['user'],
+            'name': columns[1].contents[0].strip(),
+            'resource_type': TYPES[columns[1].contents[4]['class'][0]],
+            'region_name': columns[1].contents[2],
+            'level': columns[2].string,
+            'workers': re.sub(r'\/[0-9]*$', '', columns[3].string),
+            'wage': int(columns[4].string.replace('%', '')),
+            'experience': int(columns[5].string),
+        })
+    return factories, bool(len(factories_tree) >= 25)
 
 def parse_date(date_string):
     """Parse date to object"""

app/database.py

@@ -2,14 +2,49 @@
 
 from datetime import datetime
 
-from app import session
-from app.models import State, Player
+from app import Session
+from app.models import State, Player, Factory, FactoryTrack, FactoryStat
 
 
 def get_state(state_id):
     """Get regions from state"""
-    return session.query(State).get(state_id)
+    session = Session()
+    state = session.query(State).get(state_id)
+    session.close()
+    return state
 
-def save_factories(factories):
+def save_factories(state_id, factories):
     """Save factories to database"""
-    return
+    session = Session()
+    session.close()
+
+    factory_track = FactoryTrack()
+    factory_track.state_id = state_id
+    factory_track.date_time = datetime.now()
+    session.add(factory_track)
+
+    for factory_dict in factories:
+        factory = session.query(Factory).get(factory_dict['id'])
+        if factory is None:
+            factory = save_factory(session, factory_dict)
+        factory_stat = FactoryStat()
+        factory_stat.level = factory_dict['level']
+        factory_stat.experience = factory_dict['experience']
+        factory_stat.wage = factory_dict['wage']
+        factory_stat.workers = factory_dict['workers']
+        factory_stat.factory_id = factory_dict['id']
+        factory_stat.factory_track_id = factory_track.id
+        session.add(factory_stat)
+
+    session.commit()
+    session.close()
+
+
+def save_factory(session, factory_dict):
+    """Save factory to database"""
+    factory = Factory()
+    factory.id = factory_dict['id']
+    factory.name = factory_dict['name']
+    factory.resource_type = factory_dict['resource_type']
+    session.add(factory)
+    return factory

app/models.py

@@ -14,10 +14,11 @@ class Factory(Base):
     __tablename__ = 'factory'
     id = Column(Integer, primary_key=True)
     name = Column(String)
+    resource_type = Column(SmallInteger)
 
     player_id = Column(BigInteger, ForeignKey('player.id'))
     player = relationship(
-        'User',
+        'Player',
         backref=backref('factories', lazy='dynamic')
     )
 
@@ -53,7 +54,7 @@ class FactoryStat(Base):
 
     factory_id = Column(Integer, ForeignKey('factory.id'))
     factory = relationship(
-        'FactoryTrack',
+        'Factory',
         backref=backref('factory_stats', lazy='dynamic')
     )
     factory_track_id = Column(Integer, ForeignKey('factory_track.id'))
@@ -68,12 +69,6 @@ class State(Base):
     __tablename__ = 'state'
     id = Column(Integer, primary_key=True)
     name = Column(String)
-    regions = relationship(
-        'Region',
-        secondary='state_region',
-        backref=backref('states', lazy='dynamic'),
-        lazy='dynamic'
-    )
 
 
 class Region(Base):