Add work permits, fix issues with api

app/__main__.py

@@ -3,16 +3,18 @@
 import time
 
 from app import scheduler, session, LOGGER
-from app.api import get_citizens, get_residents
-from app.database import get_state_regions, save_citizens, save_residents
+from app.api import get_citizens, get_residents, get_work_permits
+from app.database import get_state_regions, save_citizens, save_residents, save_work_permits
 
 
 def print_players(players):
     """Print professors"""
     for player in players:
-        print('{:20} {:30}'.format(
+        print('{:20} {:30} {:20} {:30}'.format(
             player['id'],
             player['name'],
+            player['registration_date'].strftime('%d-%m-%Y'),
+            player['nation'],
         ))
 
 def job_update_citizens(state_id):
@@ -39,6 +41,15 @@ def job_update_residents(state_id):
         save_residents(region.id, residents)
         LOGGER.info('"%s": done saving residents', region.name)
 
+def job_update_work_permits(state_id):
+    """Update citizens"""
+    LOGGER.info('"%s": get work permits ', state_id)
+    work_permits = get_work_permits(state_id)
+    LOGGER.info('"%s": "%s" work permits', state_id, len(work_permits))
+    print_players(work_permits)
+    save_work_permits(state_id, work_permits)
+    LOGGER.info('"%s": done saving work_permits', state_id)
+
 
 def add_update_citizens(state_id):
     """Add jobs"""
@@ -67,6 +78,7 @@ if __name__ == '__main__':
     # jobs
     # job_update_citizens(2788)
     # job_update_residents(2788)
+    job_update_work_permits(2788)
 
     # Verenigde Nederlanden
     add_update_citizens(2788)

app/api.py

@@ -1,6 +1,7 @@
 """API module"""
 
 import re
+from datetime import datetime, date, timedelta
 
 import requests
 from bs4 import BeautifulSoup
@@ -15,27 +16,32 @@ def get_citizens(region_id):
     """Get citizens from region"""
     return download_players(region_id, 'region')
 
+def get_work_permits(state_id):
+    """Get work permits from state"""
+    return download_players(state_id, 'permits_state')
+
 def download_players(region_id, player_type):
     """Download the players"""
     players = []
     more = True
     page = 0
     while more:
+        if player_type == 'permits_state':
+            url = '{}listed/{}/{}/{}'.format(BASE_URL, player_type, region_id, page*25)
+        else:
+            url = '{}listed/{}/{}/0/{}'.format(BASE_URL, player_type, region_id, page*25)
         response = requests.get(
-            '{}listed/{}/{}/0/{}'.format(BASE_URL, player_type, region_id, page*25),
+            url,
             headers=HEADERS
         )
-        tmp_players, more = parse_players(response.text)
+        if player_type == 'permits_state':
+            tmp_players, more = parse_work_permits(response.text)
+        else:
+            tmp_players, more = parse_players(response.text)
         players = players + tmp_players
         page += 1
     return players
 
-def read_citizens():
-    """Read from department file"""
-    with open('citizens.html') as file:
-        citizens = parse_players(file)
-        return citizens
-
 def parse_players(html):
     """Parse html return players"""
     soup = BeautifulSoup(html, 'html.parser')
@@ -43,9 +49,35 @@ def parse_players(html):
     players = []
     for player_tree in players_tree:
         columns = player_tree.find_all('td')
+        registration_date = parse_date(columns[2].string)
         players.append({
             'id': int(re.sub(r'^.*\/', '', columns[1]['action'])),
             'name': columns[1].string.strip(),
+            'nation': columns[5]['title'],
+            'registration_date': registration_date
+        })
+    return players, bool(len(players_tree) >= 25)
+
+def parse_work_permits(html):
+    """Parse html return players"""
+    soup = BeautifulSoup(html, 'html.parser')
+    players_tree = soup.find_all(class_='list_link')
+    players = []
+    for player_tree in players_tree:
+        columns = player_tree.find_all('td')
+        registration_date = parse_date(columns[2].string)
+        players.append({
+            'id': int(columns[1].find(class_='small')['user']),
+            'name': columns[1].contents[0].strip(),
+            'nation': columns[5]['title'],
+            'registration_date': registration_date
         })
-    more = bool(soup.find(class_='more'))
-    return players, more
+    return players, bool(len(players_tree) >= 25)
+
+def parse_date(date_string):
+    """Parse date to object"""
+    if 'Today' in date_string:
+        return date.today()
+    if 'Yesterday' in date_string:
+        return date.today() - timedelta(1)
+    return datetime.strptime(date_string, '%d %B %Y').date()

app/database.py

@@ -3,25 +3,21 @@
 from datetime import datetime
 
 from app import session
-from app.models import State, Region, Player, StateRegion, PlayerLocation, PlayerResidency
+from app.models import State, Player, StateRegion, \
+     PlayerLocation, PlayerResidency, StateWorkPermit
 
 
 def get_state_regions(state_id):
     """Get regions from state"""
     state = session.query(State).get(state_id)
-    regions = state.regions.filter(StateRegion.until_date_time == None).all()
-    return regions
+    return state.regions.filter(StateRegion.until_date_time == None).all()
 
 def save_citizens(region_id, citizens):
     """Save citizens to database"""
     for player_dict in citizens:
         player = session.query(Player).get(player_dict['id'])
         if player is None:
-            player = Player()
-            player.id = player_dict['id']
-            player.name = player_dict['name']
-            session.add(player)
-            session.commit()
+            player = save_player(player_dict)
         last_region = player.locations.first()
         if not last_region or last_region.id != region_id:
             player_location = PlayerLocation()
@@ -33,7 +29,8 @@ def save_citizens(region_id, citizens):
             if last_region:
                 old_player_location = session.query(PlayerLocation) \
                     .filter(PlayerLocation.player_id == player.id) \
-                    .filter(PlayerLocation.region_id == last_region.id).first()
+                    .filter(PlayerLocation.region_id == last_region.id) \
+                    .order_by(PlayerLocation.from_date_time.desc()).first()
                 old_player_location.until_date_time = datetime.now()
                 session.commit()
 
@@ -42,11 +39,7 @@ def save_residents(region_id, residents):
     for player_dict in residents:
         player = session.query(Player).get(player_dict['id'])
         if player is None:
-            player = Player()
-            player.id = player_dict['id']
-            player.name = player_dict['name']
-            session.add(player)
-            session.commit()
+            player = save_player(player_dict)
         last_residency = player.residencies.first()
         if not last_residency or last_residency.id != region_id:
             player_location = PlayerResidency()
@@ -58,6 +51,41 @@ def save_residents(region_id, residents):
             if last_residency:
                 old_player_residency = session.query(PlayerResidency) \
                     .filter(PlayerResidency.player_id == player.id) \
-                    .filter(PlayerResidency.region_id == last_residency.id).first()
+                    .filter(PlayerResidency.region_id == last_residency.id) \
+                    .order_by(PlayerResidency.from_date_time.desc()).first()
                 old_player_residency.until_date_time = datetime.now()
                 session.commit()
+
+
+def save_work_permits(state_id, work_permits):
+    """Save residents to database"""
+    for player_dict in work_permits:
+        player = session.query(Player).get(player_dict['id'])
+        if player is None:
+            player = save_player(player_dict)
+        last_work_permit = player.state_work_permits.first()
+        if not last_work_permit or last_work_permit.id != state_id:
+            state_work_permit = StateWorkPermit()
+            state_work_permit.player_id = player.id
+            state_work_permit.state_id = state_id
+            state_work_permit.from_date_time = datetime.now()
+            session.add(state_work_permit)
+            session.commit()
+            if last_work_permit:
+                old_state_work_permit = session.query(StateWorkPermit) \
+                    .filter(StateWorkPermit.player_id == player.id) \
+                    .filter(StateWorkPermit.state_id == last_work_permit.id) \
+                    .order_by(StateWorkPermit.from_date_time.desc()).first()
+                old_state_work_permit.until_date_time = datetime.now()
+                session.commit()
+
+def save_player(player_dict):
+    """Save player to database"""
+    player = Player()
+    player.id = player_dict['id']
+    player.name = player_dict['name']
+    player.nation = player_dict['nation']
+    player.registration_date = player_dict['registration_date']
+    session.add(player)
+    session.commit()
+    return player

app/models.py

@@ -1,7 +1,7 @@
 """Database models"""
 
-from sqlalchemy import MetaData, Column, ForeignKey, Integer, String, \
-    SmallInteger, DateTime, BigInteger
+from sqlalchemy import Column, ForeignKey, Integer, String, \
+    DateTime, BigInteger, Date
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.ext.declarative import declarative_base
 
@@ -35,6 +35,15 @@ class StateRegion(Base):
     until_date_time = Column(DateTime)
 
 
+class StateWorkPermit(Base):
+    """Model for state work permit"""
+    __tablename__ = 'state_work_permit'
+    state_id = Column(Integer, ForeignKey('state.id'), primary_key=True)
+    player_id = Column(BigInteger, ForeignKey('player.id'), primary_key=True)
+    from_date_time = Column(DateTime)
+    until_date_time = Column(DateTime)
+
+
 class State(Base):
     """Model for state"""
     __tablename__ = 'state'
@@ -66,6 +75,7 @@ class Player(Base):
     id = Column(BigInteger, primary_key=True)
     name = Column(String)
     nation = Column(String)
+    registration_date = Column(Date)
     residencies = relationship(
         'Region',
         secondary='player_residency',
@@ -80,3 +90,9 @@ class Player(Base):
         lazy='dynamic',
         order_by='desc(PlayerLocation.from_date_time)'
     )
+    state_work_permits = relationship(
+        'State',
+        secondary='state_work_permit',
+        lazy='dynamic',
+        order_by='desc(StateWorkPermit.from_date_time)'
+    )