Improved application

app/__init__.py

@@ -14,18 +14,19 @@ from app.models import Base, State, Region, Player
 load_dotenv()
 
 # database
-engine = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
-Session = sessionmaker(bind=engine)
+ENGINE = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
+SESSION = sessionmaker(bind=ENGINE)
 
 # scheduler
-scheduler = BackgroundScheduler()
-scheduler.start()
+SCHEDULER = BackgroundScheduler(
+    daemon=True,
+    job_defaults={'misfire_grace_time': 900},
+)
+SCHEDULER.start()
 
 # module logger
 LOGGER = logging.getLogger(__name__)
 LOGGER.setLevel(logging.INFO)
-
-# scheduler logger
 SCHEDULER_LOGGER = logging.getLogger('apscheduler')
 SCHEDULER_LOGGER.setLevel(logging.DEBUG)
 
@@ -45,7 +46,6 @@ FILE_HANDLER.setFormatter(FORMATTER)
 # add the handlers to logger
 LOGGER.addHandler(STREAM_HANDLER)
 LOGGER.addHandler(FILE_HANDLER)
-
 SCHEDULER_LOGGER.addHandler(STREAM_HANDLER)
 SCHEDULER_LOGGER.addHandler(FILE_HANDLER)
 

app/__main__.py

@@ -1,111 +1,70 @@
 """Main app"""
 
+import sys
 import time
 
-from app import scheduler, LOGGER
-from app.api import get_citizens, get_residents, get_work_permits
-from app.database import get_state_regions, save_citizens, save_residents, save_work_permits
+from app import SCHEDULER, LOGGER, jobs
 
 
-def print_players(players):
-    """Print professors"""
-    for player in players:
-        print('{:20} {:30} {:30}'.format(
-            player['id'],
-            player['name'],
-            player['nation'],
-        ))
-
-def job_update_citizens(state_id):
-    """Update citizens"""
-    LOGGER.info('Run update citizens for state "%s"', state_id)
-    regions = get_state_regions(state_id)
-    for region in regions:
-        LOGGER.info('regio %6s: get citizens', region.id)
-        citizens = get_citizens(region.id)
-        LOGGER.info('regio %6s: "%s" citizens', region.id, len(citizens))
-        # print_players(citizens)
-        save_citizens(region.id, citizens)
-        LOGGER.info('regio %6s: done saving citizens', region.id)
+if __name__ == '__main__':
+    LOGGER.info('Starting application')
 
-def job_update_residents(state_id):
-    """Update residents"""
-    LOGGER.info('Run update residents for state "%s"', state_id)
-    regions = get_state_regions(state_id)
-    for region in regions:
-        LOGGER.info('regio %6s: get residents', region.id)
-        residents = get_residents(region.id)
-        LOGGER.info('regio %6s: "%s" residents ', region.id, len(residents))
-        # print_players(residents)
-        save_residents(region.id, residents)
-        LOGGER.info('regio %6s: done saving residents', region.id)
+    # state to track citizens, residents, and work permits
+    STATE_IDS = [
+        3304, # VN
+        3261, # Craftbroec
+    ]
 
-def job_update_work_permits(state_id):
-    """Update citizens"""
-    LOGGER.info('state "%s": get work permits ', state_id)
-    work_permits = get_work_permits(state_id)
-    LOGGER.info('state "%s": "%s" work permits', state_id, len(work_permits))
-    # print_players(work_permits)
-    save_work_permits(state_id, work_permits)
-    LOGGER.info('state "%s": done saving work_permits', state_id)
+    # aditional regions to track citizens and residents
+    REGION_IDS = [
+        4001, # Noord
+        4002, # Oost
+        4003, # West
+        4004, # Zuid
+        4008, # Amsterdam
+        4101, # Vlaanderen
+        4102, # Walonie
+        4103, # Brussel
+        200062, # Maan regio 62
+    ]
 
+    # jobs
+    jobs.update_citizens(STATE_IDS, REGION_IDS)
+    jobs.update_residents(STATE_IDS, REGION_IDS)
+    jobs.update_work_permits(STATE_IDS)
 
-def add_update_citizens(state_id):
-    """Add jobs"""
-    scheduler.add_job(
-        job_update_citizens,
+    # Update citizens
+    SCHEDULER.add_job(
+        jobs.update_citizens,
         'cron',
-        args=[state_id],
-        id='citizens_{}'.format(state_id),
+        args=[STATE_IDS, REGION_IDS],
+        id='update_citizens',
         replace_existing=True,
         hour='1,3,5,7,9,11,13,15,17,19,21,23'
     )
-
-def add_update_residents(state_id):
-    """Add jobs"""
-    scheduler.add_job(
-        job_update_residents,
+    # Update residents
+    SCHEDULER.add_job(
+        jobs.update_residents,
         'cron',
-        args=[state_id],
-        id='residents_{}'.format(state_id),
+        args=[STATE_IDS, REGION_IDS],
+        id='residents',
         replace_existing=True,
-        hour='1,7,13,19'
+        hour='1,4,7,10,13,16,19,22'
     )
-
-def add_update_work_permits(state_id):
-    """Add jobs"""
-    scheduler.add_job(
-        job_update_work_permits,
+    # Work permits
+    SCHEDULER.add_job(
+        jobs.update_work_permits,
         'cron',
-        args=[state_id],
-        id='work_permits_{}'.format(state_id),
+        args=[STATE_IDS],
+        id='work_permits',
         replace_existing=True,
-        hour='4'
+        hour='2,5,8,11,14,17,20,23'
     )
 
-if __name__ == '__main__':
-    LOGGER.info('Starting application')
-    # jobs
-    # job_update_citizens(2788)
-    # job_update_residents(2788)
-    # job_update_work_permits(2788)
-
-    # Verenigde Nederlanden
-    add_update_citizens(2788)
-    add_update_residents(2788)
-    add_update_work_permits(2788)
-    # Belgium
-    add_update_citizens(2604)
-    add_update_residents(2604)
-    add_update_work_permits(2604)
-    # De Provincien
-    add_update_citizens(2620)
-    add_update_residents(2620)
-    add_update_work_permits(2620)
-
     try:
         while True:
             time.sleep(100)
     except KeyboardInterrupt:
         LOGGER.info('Exiting application')
-        exit()
+        SCHEDULER.shutdown()
+        sys.exit()

app/app.py

@@ -0,0 +1,52 @@
+"""General application logic"""
+
+from app import LOGGER, api, database
+
+
+def print_players(players):
+    """Print professors"""
+    for player in players:
+        print('{:20} {:30} {:30}'.format(
+            player['id'],
+            player['name'],
+            player['nation'],
+        ))
+
+def update_citizens(state_ids, region_ids):
+    """Update citizens"""
+    regions = database.get_regions(region_ids)
+    for state_id in state_ids:
+        regions += database.get_state_regions(state_id)
+    LOGGER.info('update citizens for "%s" regions', len(regions))
+    for region in regions:
+        LOGGER.info('regio %6s: get citizens', region.id)
+        citizens = api.get_citizens(region.id)
+        LOGGER.info('regio %6s: "%s" citizens', region.id, len(citizens))
+        # print_players(citizens)
+        database.save_citizens(region.id, citizens)
+        LOGGER.info('regio %6s: done saving citizens', region.id)
+
+def update_residents(state_ids, region_ids):
+    """Update residents"""
+    regions = database.get_regions(region_ids)
+    for state_id in state_ids:
+        regions += database.get_state_regions(state_id)
+    LOGGER.info('update residents for "%s" regions', len(regions))
+    for region in regions:
+        LOGGER.info('regio %6s: get residents', region.id)
+        residents = api.get_residents(region.id)
+        LOGGER.info('regio %6s: "%s" residents ', region.id, len(residents))
+        # print_players(residents)
+        database.save_residents(region.id, residents)
+        LOGGER.info('regio %6s: done saving residents', region.id)
+
+def update_work_permits(state_ids):
+    """Update work permits"""
+    LOGGER.info('update work permits for "%s" states', len(state_ids))
+    for state_id in state_ids:
+        LOGGER.info('state %6s: get work permits ', state_id)
+        work_permits = api.get_work_permits(state_id)
+        LOGGER.info('state %6s: "%s" work permits', state_id, len(work_permits))
+        # print_players(work_permits)
+        database.save_work_permits(state_id, work_permits)
+        LOGGER.info('state %6s: done saving work_permits', state_id)

app/database.py

@@ -2,22 +2,39 @@
 
 from datetime import datetime
 
-from app import Session, LOGGER
-from app.models import State, Player, StateRegion, \
+from app import SESSION, LOGGER
+from app.models import State, Region, Player, StateRegion, \
      PlayerLocation, PlayerResidency, StateWorkPermit
 
 
+def get_regions(region_ids):
+    """Get regions from list"""
+    session = SESSION()
+    regions = []
+    for region_id in region_ids:
+        region = session.query(Region).get(region_id)
+        if not region:
+            region = save_region(session, region_id)
+        regions.append(region)
+    session.close()
+    return regions
+
 def get_state_regions(state_id):
     """Get regions from state"""
-    session = Session()
+    session = SESSION()
     state = session.query(State).get(state_id)
-    regions = state.regions.filter(StateRegion.until_date_time == None).all()
+    region_list = []
+    if state:
+        region_list = state.regions.filter(StateRegion.until_date_time == None).all()
+    else:
+        save_state(session, state_id)
+        LOGGER.error('State %6s: not found', state_id)
     session.close()
-    return regions
+    return region_list
 
 def save_citizens(region_id, citizens):
     """Save citizens to database"""
-    session = Session()
+    session = SESSION()
     player_ids = []
     new_citizens = 0
     for player_dict in citizens:
@@ -25,8 +42,11 @@ def save_citizens(region_id, citizens):
         if player is None:
             player = save_player(session, player_dict)
         player_ids.append(player.id)
-        last_region = player.locations.first()
-        if not last_region or last_region.id != region_id:
+        last_location = player.locations \
+            .filter(PlayerLocation.region_id == region_id) \
+            .filter(PlayerLocation.until_date_time == None) \
+            .first()
+        if not last_location:
             new_citizens += 1
             player_location = PlayerLocation()
             player_location.player_id = player.id
@@ -50,21 +70,27 @@ def save_citizens(region_id, citizens):
 
 def save_residents(region_id, residents):
     """Save residents to database"""
-    session = Session()
+    session = SESSION()
     player_ids = []
+    new_residents = 0
     for player_dict in residents:
         player = session.query(Player).get(player_dict['id'])
         if player is None:
             player = save_player(session, player_dict)
         player_ids.append(player.id)
-        last_residency = player.residencies.first()
-        if not last_residency or last_residency.id != region_id:
+        last_residency = player.residencies \
+            .filter(PlayerResidency.region_id == region_id) \
+            .filter(PlayerResidency.until_date_time == None) \
+            .first()
+        if not last_residency:
+            new_residents += 1
             player_location = PlayerResidency()
             player_location.player_id = player.id
             player_location.region_id = region_id
             player_location.from_date_time = datetime.now().replace(second=0, minute=0)
             session.add(player_location)
-            session.commit()
+    LOGGER.info('regio %6s: "%s" new residents', region_id, new_residents)
+    session.commit()
     current_residents = session.query(PlayerResidency) \
         .filter(PlayerResidency.region_id == region_id) \
         .filter(PlayerResidency.until_date_time == None).all()
@@ -77,21 +103,27 @@ def save_residents(region_id, residents):
 
 def save_work_permits(state_id, work_permits):
     """Save residents to database"""
-    session = Session()
+    session = SESSION()
     player_ids = []
+    new_work_permits = 0
     for player_dict in work_permits:
         player = session.query(Player).get(player_dict['id'])
         if player is None:
             player = save_player(session, player_dict)
         player_ids.append(player.id)
-        last_work_permit = player.state_work_permits.first()
-        if not last_work_permit or last_work_permit.id != state_id:
+        last_work_permit = player.state_work_permits \
+            .filter(StateWorkPermit.state_id == state_id) \
+            .filter(StateWorkPermit.until_date_time == None) \
+            .first()
+        if not last_work_permit:
+            new_residents += 1
             state_work_permit = StateWorkPermit()
             state_work_permit.player_id = player.id
             state_work_permit.state_id = state_id
             state_work_permit.from_date_time = player_dict['from']
             session.add(state_work_permit)
-            session.commit()
+    session.commit()
+    LOGGER.info('state %6s: "%s" new work permits', state_id, new_work_permits)
     current_work_permits = session.query(StateWorkPermit) \
         .filter(StateWorkPermit.state_id == state_id) \
         .filter(StateWorkPermit.until_date_time == None).all()
@@ -112,3 +144,19 @@ def save_player(session, player_dict):
     session.add(player)
     session.commit()
     return player
+
+def save_state(session, state_id):
+    """Save state to database"""
+    state = State()
+    state.id = state_id
+    session.add(state)
+    session.commit()
+    return state
+
+def save_region(session, region_id):
+    """Save region to database"""
+    region = Region()
+    region.id = region_id
+    session.add(region)
+    session.commit()
+    return region

app/jobs.py

@@ -0,0 +1,16 @@
+"""Jobs for scheduler module"""
+
+from app import app
+
+
+def update_citizens(state_ids, region_ids):
+    """Update citizens"""
+    app.update_citizens(state_ids, region_ids)
+
+def update_residents(state_ids, region_ids):
+    """Update residents"""
+    app.update_residents(state_ids, region_ids)
+
+def update_work_permits(state_ids):
+    """Update work permits"""
+    app.update_work_permits(state_ids)